{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Streetcar Delay Prediction - Deep Learning - REFACTORED\n",
    "\n",
    "GOAL: predict streetcar delays using a simple Keras model\n",
    "\n",
    "Refactored to look at delays by hour by day by route by direction\n",
    "\n",
    "Source dataset: : https://open.toronto.ca/dataset/ttc-streetcar-delay-data/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Links to key parts of the notebook <a name='linkanchor' />\n",
    "<a href=#ingestdash>Ingest data</a>\n",
    "\n",
    "<a href=#definecategories>Define feature categories</a>\n",
    "\n",
    "<a href=#modelfit>Define and fit model</a>\n",
    "\n",
    "<a href=#reload>Reload saved model and weights</a>\n",
    "\n",
    "<a href=#confusionmatrix>Confusion matrix</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Common imports and global variable definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# common imports\n",
    "import zipfile\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import seaborn as sns\n",
    "# import datetime, timedelta\n",
    "import datetime\n",
    "import pydotplus\n",
    "from datetime import datetime, timedelta\n",
    "from datetime import date\n",
    "from dateutil import relativedelta\n",
    "from io import StringIO\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from pickle import dump\n",
    "from pickle import load\n",
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.base import TransformerMixin\n",
    "# DSX code to import uploaded documents\n",
    "from io import StringIO\n",
    "import requests\n",
    "import json\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import os\n",
    "import yaml\n",
    "import math\n",
    "import sys\n",
    "from subprocess import check_output\n",
    "from IPython.display import display\n",
    "#model libraries\n",
    "from tensorflow.keras.metrics import Accuracy, Recall, Precision\n",
    "from tensorflow.keras.layers import Input, Dropout, Dense, BatchNormalization, Activation, concatenate, GRU, Embedding, Flatten, BatchNormalization\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.layers import BatchNormalization\n",
    "#from tf.keras.layers.normalization import BatchNormalization\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.optimizers import SGD\n",
    "from tensorflow.keras import backend as K\n",
    "# from tensorflow.keras.utils.vis_utils import plot_model\n",
    "from tensorflow.keras.utils import plot_model\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow.python.keras.callbacks import TensorBoard\n",
    "#import datetime\n",
    "#from datetime import date\n",
    "from sklearn import metrics\n",
    "# import pipeline libraries\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.base import TransformerMixin\n",
    "from sklearn.base import BaseEstimator\n",
    "from custom_classes import encode_categorical\n",
    "from custom_classes import prep_for_keras_input\n",
    "from custom_classes import fill_empty\n",
    "from custom_classes import encode_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current directory is: C:\\personal\\manning\\deep_learning_for_structured_data\\notebooks\n",
      "path_to_yaml C:\\personal\\manning\\deep_learning_for_structured_data\\notebooks\\streetcar_model_training_config.yml\n"
     ]
    }
   ],
   "source": [
    "# load config file\n",
    "current_path = os.getcwd()\n",
    "print(\"current directory is: \"+current_path)\n",
    "\n",
    "path_to_yaml = os.path.join(current_path, 'streetcar_model_training_config.yml')\n",
    "print(\"path_to_yaml \"+path_to_yaml)\n",
    "try:\n",
    "    with open (path_to_yaml, 'r') as c_file:\n",
    "        config = yaml.safe_load(c_file)\n",
    "except Exception as e:\n",
    "    print('Error reading the config file')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date today 2020-05-16 14:57:44.239777\n",
      "start date 2014-01-01\n",
      "end date 2019-12-31\n"
     ]
    }
   ],
   "source": [
    "# load parameters\n",
    "\n",
    "repeatable_run = config['test_parms']['repeatable_run']\n",
    "# fix seeds to get identical results on mulitiple runs\n",
    "if repeatable_run:\n",
    "    from numpy.random import seed\n",
    "    seed(4)\n",
    "    tf.random.set_seed(7)\n",
    "\n",
    "\n",
    "testproportion = config['test_parms']['testproportion'] # proportion of data reserved for test set\n",
    "trainproportion = config['test_parms']['trainproportion'] # proportion of non-test data dedicated to training (vs. validation)\n",
    "get_test_train_acc = config['test_parms']['get_test_train_acc']\n",
    "verboseout = config['general']['verboseout']\n",
    "includetext = config['general']['includetext'] # switch to determine whether text columns are included in the model\n",
    "save_model_plot = config['general']['save_model_plot'] # switch to determine whether to generate plot with plot_model\n",
    "tensorboard_callback = config['general']['tensorboard_callback'] # switch to determine if tensorboard callback defined\n",
    "\n",
    "presaved = config['general']['presaved']\n",
    "savemodel = config['general']['savemodel']\n",
    "picklemodel = config['general']['picklemodel']\n",
    "hctextmax = config['general']['hctextmax']\n",
    "maxwords = config['general']['maxwords']\n",
    "textmax = config['general']['textmax']\n",
    "\n",
    "targetthresh = config['general']['targetthresh']\n",
    "targetcontinuous = config['general']['targetcontinuous']\n",
    "\n",
    "#time of day thresholds\n",
    "time_of_day = {'overnight':{'start':0,'end':5},'morning_rush':{'start':5,'end':10},\n",
    "              'midday':{'start':10,'end':15},'aft_rush':{'start':15,'end':19},'evening':{'start':19,'end':24}}\n",
    "\n",
    "\n",
    "\n",
    "emptythresh = config['general']['emptythresh']\n",
    "zero_weight = config['general']['zero_weight']\n",
    "one_weight = config['general']['one_weight']\n",
    "one_weight_offset = config['general']['one_weight_offset']\n",
    "patience_threshold = config['general']['patience_threshold']\n",
    "\n",
    "\n",
    "# modifier for saved model elements\n",
    "modifier = config['general']['modifier']\n",
    "\n",
    "# control whether training controlled by early stop\n",
    "early_stop = True\n",
    "\n",
    "# default hyperparameter values\n",
    "learning_rate = config['hyperparameters']['learning_rate']\n",
    "dropout_rate = config['hyperparameters']['dropout_rate']\n",
    "l2_lambda = config['hyperparameters']['l2_lambda']\n",
    "loss_func = config['hyperparameters']['loss_func']\n",
    "output_activation = config['hyperparameters']['output_activation']\n",
    "batch_size = config['hyperparameters']['batch_size']\n",
    "epochs = config['hyperparameters']['epochs']\n",
    "\n",
    "# date values\n",
    "date_today = datetime.now()\n",
    "print(\"date today\",date_today)\n",
    "start_date =  date(config['general']['start_year'],config['general']['start_month'], config['general']['start_day'])\n",
    "print(\"start date\",start_date)\n",
    "end_date = date(config['general']['end_year'],config['general']['end_month'], config['general']['end_day'])\n",
    "print(\"end date\",end_date)\n",
    "\n",
    "\n",
    "# pickled original dataset and post-preprocessing dataset\n",
    "pickled_data_file = config['general']['pickled_data_file']\n",
    "pickled_dataframe = config['general']['pickled_dataframe']\n",
    "routedirection_file = config['general']['route_direction_file']\n",
    "\n",
    "# experiment parameter\n",
    "\n",
    "current_experiment = config['test_parms']['current_experiment']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# time_of_day = {'overnight':{'start':0,'end':5},'morning_rush':{'start':5,'end':10},\n",
    "#              'midday':{'start':10,'end':15},'aft_rush':{'start':15,'end':19},'evening':{'start':19,'end':23}}\n",
    "\n",
    "\n",
    "def get_time(hour):\n",
    "    for tod in time_of_day:\n",
    "        if (hour >= time_of_day[tod]['start']) and (hour < time_of_day[tod]['end']):\n",
    "            tod_out = tod\n",
    "    return(tod_out)\n",
    "\n",
    "def weekend_time(day, tod):\n",
    "    if (day=='Saturday') or (day=='Sunday'):\n",
    "        return('w'+tod)\n",
    "    else:\n",
    "        return(tod)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the paths required\n",
    "\n",
    "def get_path():\n",
    "    '''get the path for data files'''\n",
    "    rawpath = os.getcwd()\n",
    "    # data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n",
    "    path = os.path.abspath(os.path.join(rawpath, '..', 'data'))\n",
    "    return(path)\n",
    "\n",
    "def get_pipeline_path():\n",
    "    '''get the path for data files'''\n",
    "    rawpath = os.getcwd()\n",
    "    # data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n",
    "    path = os.path.abspath(os.path.join(rawpath, '..', 'pipelines'))\n",
    "    return(path)\n",
    "\n",
    "def get_model_path():\n",
    "    '''get the path for data files'''\n",
    "    rawpath = os.getcwd()\n",
    "    # data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n",
    "    path = os.path.abspath(os.path.join(rawpath, '..', 'models'))\n",
    "    return(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_experiment_parameters(experiment_number, count_no_delay, count_delay):\n",
    "    ''' set the appropriate parameters for the experiment '''\n",
    "    print(\"setting parameters for experiment \", experiment_number)\n",
    "    # default settings for early stopping:\n",
    "    es_monitor = \"val_loss\"\n",
    "    es_mode = \"min\"\n",
    "    if experiment_number == 0:\n",
    "        #\n",
    "        early_stop = False\n",
    "        #\n",
    "        one_weight = 1.0\n",
    "        #\n",
    "        epochs = 1\n",
    "    elif experiment_number == 9:\n",
    "        #\n",
    "        early_stop = True\n",
    "        es_monitor=\"val_accuracy\"\n",
    "        es_mode = \"max\"\n",
    "        #\n",
    "        one_weight = (count_no_delay/count_delay) + one_weight_offset\n",
    "        #\n",
    "        get_test_train_acc = False\n",
    "        #\n",
    "        epochs = 20    \n",
    "    elif experiment_number == 1:\n",
    "        #\n",
    "        early_stop = False\n",
    "        #\n",
    "        one_weight = 1.0\n",
    "        #\n",
    "        epochs = 10\n",
    "    elif experiment_number == 2:\n",
    "        #\n",
    "        early_stop = False\n",
    "        #\n",
    "        one_weight = 1.0\n",
    "        #\n",
    "        epochs = 50\n",
    "    elif experiment_number == 3:\n",
    "        #\n",
    "        early_stop = False\n",
    "        #\n",
    "        one_weight = (count_no_delay/count_delay) + one_weight_offset\n",
    "        #\n",
    "        epochs = 50\n",
    "    elif experiment_number == 4:\n",
    "        #\n",
    "        early_stop = True\n",
    "        es_monitor = \"val_loss\"\n",
    "        es_mode = \"min\"\n",
    "        #\n",
    "        one_weight = (count_no_delay/count_delay) + one_weight_offset\n",
    "        #\n",
    "        epochs = 50\n",
    "    elif experiment_number == 5:\n",
    "        #\n",
    "        early_stop = True\n",
    "        # if early stopping fails because the level of TensorFlow/Python, comment out the following\n",
    "        # line and uncomment the subsequent if statement\n",
    "        es_monitor=\"val_accuracy\"\n",
    "        '''\n",
    "        if sys.version_info >= (3,7):\n",
    "            es_monitor=\"val_accuracy\"\n",
    "        else:\n",
    "            es_monitor = \"val_acc\"\n",
    "        '''\n",
    "        es_mode = \"max\"\n",
    "        #\n",
    "        one_weight = (count_no_delay/count_delay) + one_weight_offset\n",
    "        #\n",
    "        epochs = 50\n",
    "    else:\n",
    "        early_stop = True\n",
    "    return(early_stop, one_weight, epochs,es_monitor,es_mode)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ingest data and create refactored dataframe\n",
    "- Ingest data for route information and delay information\n",
    "- Create refactored dataframe with one row per route / direction / timeslot combination\n",
    "\n",
    "<a name='ingestdash' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load list of valid routes and directions into dataframe\n",
    "def ingest_data(path):\n",
    "    routedirection_frame = pd.read_csv(os.path.join(path,routedirection_file))\n",
    "    routedirection_frame.tail()\n",
    "    file_name = os.path.join(path,pickled_dataframe)\n",
    "    merged_data = pd.read_pickle(file_name)\n",
    "    merged_data.head()\n",
    "    return(routedirection_frame, merged_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add derived columns to merged_data dataframe\n",
    "def prep_merged_data(merged_data):\n",
    "    # define cols for year month day hour\n",
    "    merged_data['year'] = pd.DatetimeIndex(merged_data['Report Date']).year\n",
    "    merged_data['month'] = pd.DatetimeIndex(merged_data['Report Date']).month\n",
    "    merged_data['daym'] = pd.DatetimeIndex(merged_data['Report Date']).day\n",
    "    merged_data['hour'] = pd.DatetimeIndex(merged_data['Report Date Time']).hour\n",
    "    # define time of day column\n",
    "    merged_data['time_of_day'] = merged_data['hour'].apply(lambda x:get_time(x))\n",
    "    # add a special timeframe for weekends\n",
    "    merged_data['time_of_day'] = merged_data.apply(lambda x: weekend_time(x['Day'], x['time_of_day']), axis=1)\n",
    "    if targetcontinuous:\n",
    "        merged_data['target'] = merged_data['Min Delay']\n",
    "    else:\n",
    "        merged_data['target'] = np.where(merged_data['Min Delay'] >= targetthresh, 1, 0 )\n",
    "    return(merged_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate dataframe containing rows for each timeslot for each route for each direction\n",
    "# and merge with the input merged_data dataframe to get a result of a sparse dataframe with the\n",
    "# timeslot / route / direction combinations where delays occurred\n",
    "def prep_sparse_df(routedirection_frame, merged_data):\n",
    "    routedirection_frame['count'] = 0\n",
    "    print(\"routedirection\")\n",
    "    display(routedirection_frame[:5])\n",
    "    # define a dataframe with a row for each date to be covered\n",
    "    days = pd.date_range(start_date, end_date, freq='D')\n",
    "    date_frame = pd.DataFrame({'date':days,'count':0})\n",
    "    print(\"date_frame\")\n",
    "    display(date_frame[:5])\n",
    "    # define a dataframe with a row for each hour\n",
    "    hour_list = list(range(0,24))\n",
    "    hour_frame = pd.DataFrame({'hour':hour_list,'count':0})\n",
    "    print(\"hour_frame\")\n",
    "    display(hour_frame[:5])\n",
    "    #vprint(hour_frame.head())\n",
    "    # merge date_frame and routedirection\n",
    "    result1 = pd.merge(date_frame, routedirection_frame, on='count', how='outer')\n",
    "    print(\"result1\")\n",
    "    display(result1[:5])\n",
    "    # merge result1 with hour_frame\n",
    "    result2 = pd.merge(result1, hour_frame, on='count', how='outer')\n",
    "    result2 = result2.rename(columns={'date': 'Report Date'})\n",
    "    result2.Route = result2.Route.astype(str)\n",
    "    # segment the date\n",
    "    result2['year'] = pd.DatetimeIndex(result2['Report Date']).year\n",
    "    result2['month'] = pd.DatetimeIndex(result2['Report Date']).month\n",
    "    result2['daym'] = pd.DatetimeIndex(result2['Report Date']).day\n",
    "    result2['day'] = pd.DatetimeIndex(result2['Report Date']).weekday\n",
    "    print(\"result2\")\n",
    "    display(result2[:5])\n",
    "    print(\"merged_data before\")\n",
    "    display(merged_data[:5])\n",
    "    # drop extraneous columns from merged_data\n",
    "    merged_data = merged_data.drop(['Time',\n",
    "     'Report Date Time',\n",
    "     'year',\n",
    "     'month',\n",
    "     'daym',\n",
    "     'time_of_day','Min Gap','Location','Incident','Vehicle','target','Day'],axis=1)\n",
    "    print(\"merged_data after dropping extraneous columns\")\n",
    "    display(merged_data[:5])\n",
    "    # join result2 and the trimmed merged_data\n",
    "    result3 = pd.merge(result2,merged_data ,how='left', on=['Report Date','Route','Direction','hour'])\n",
    "    result3['Min Delay'].fillna(value=0.0,inplace=True)\n",
    "    result3['target'] = np.where(result3['Min Delay'] > 0.0, 1, 0 )\n",
    "    print(\"result3\")\n",
    "    display(result3[:5])\n",
    "    return(result3)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# REFACTORED DATAFRAME SHOULD HAVE THE FOLLOWING COLUMNS:\n",
    "# DAY - for every day in the history from Jan 1 2014 to July 31 2018\n",
    "# HOUR - for every hour of the day\n",
    "#  for 501, regular route 5:00 am - midnight; 301 overnight\n",
    "#   for 503: 7- 10:00 am; 4-7:00 pm\n",
    "# for 504 5:00 am - 2:00 am; 304 overnight\n",
    "# for 505 5:00 am - 1:00 am\n",
    "# for 506 5:00 am - 1:00 am; 306 overnight\n",
    "# for 509 5:00 am - 1:00 am\n",
    "# for 510 5:00 am - 2:00 am; 310 overnight\n",
    "# for 511 5:00 am - 1:00 am\n",
    "# for 512 5:00 am - 2:00 am\n",
    "# for 514 (Cherry street)\n",
    "# ROUTE\n",
    "# DIRECTION\n",
    "# DELAY - where this could be count OR duration OR binary\n",
    "\n",
    "# example of filling in values:\n",
    "# data['PriceDate'] =  pd.to_datetime(data['PriceDate'], format='%m/%d/%Y')\n",
    "# data = data.sort_values(by=['PriceDate'], ascending=[True])\n",
    "# data.set_index('PriceDate', inplace=True)\n",
    "# print (data)\n",
    "\n",
    "# data = data.resample('D').ffill().reset_index()\n",
    "# print (data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Master Prep Calls\n",
    "Contains calls to functions to load data, prep input dataframes, and create refactored dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "path is C:\\personal\\manning\\deep_learning_for_structured_data\\data\n",
      "shape of pre refactored dataset (61553, 17)\n",
      "routedirection\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Route</th>\n",
       "      <th>Direction</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>301</td>\n",
       "      <td>w</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>301</td>\n",
       "      <td>b</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>304</td>\n",
       "      <td>e</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>304</td>\n",
       "      <td>e</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Route Direction  count\n",
       "0    301         e      0\n",
       "1    301         w      0\n",
       "2    301         b      0\n",
       "3    304         e      0\n",
       "4    304         e      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date_frame\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2014-01-03</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014-01-04</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014-01-05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date  count\n",
       "0 2014-01-01      0\n",
       "1 2014-01-02      0\n",
       "2 2014-01-03      0\n",
       "3 2014-01-04      0\n",
       "4 2014-01-05      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hour_frame\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hour</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hour  count\n",
       "0     0      0\n",
       "1     1      0\n",
       "2     2      0\n",
       "3     3      0\n",
       "4     4      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "result1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>count</th>\n",
       "      <th>Route</th>\n",
       "      <th>Direction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>304</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>304</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date  count  Route Direction\n",
       "0 2014-01-01      0    301         e\n",
       "1 2014-01-01      0    301         w\n",
       "2 2014-01-01      0    301         b\n",
       "3 2014-01-01      0    304         e\n",
       "4 2014-01-01      0    304         e"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "result2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>count</th>\n",
       "      <th>Route</th>\n",
       "      <th>Direction</th>\n",
       "      <th>hour</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "      <th>day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>0</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>1</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>2</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>3</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>4</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Report Date  count Route Direction  hour  year  month  daym  day\n",
       "0  2014-01-01      0   301         e     0  2014      1     1    2\n",
       "1  2014-01-01      0   301         e     1  2014      1     1    2\n",
       "2  2014-01-01      0   301         e     2  2014      1     1    2\n",
       "3  2014-01-01      0   301         e     3  2014      1     1    2\n",
       "4  2014-01-01      0   301         e     4  2014      1     1    2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "merged_data before\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Day</th>\n",
       "      <th>Direction</th>\n",
       "      <th>Incident</th>\n",
       "      <th>Location</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>Min Gap</th>\n",
       "      <th>Report Date</th>\n",
       "      <th>Route</th>\n",
       "      <th>Time</th>\n",
       "      <th>Vehicle</th>\n",
       "      <th>Report Date Time</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "      <th>hour</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Report Date Time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2014-01-02 06:31:31</th>\n",
       "      <td>Thursday</td>\n",
       "      <td>e</td>\n",
       "      <td>Late Leaving Garage</td>\n",
       "      <td>dundas and roncesvalles</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>505</td>\n",
       "      <td>06:31:00</td>\n",
       "      <td>4018</td>\n",
       "      <td>2014-01-02 06:31:31</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>morning_rush</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-02 12:43:43</th>\n",
       "      <td>Thursday</td>\n",
       "      <td>e</td>\n",
       "      <td>Utilized Off Route</td>\n",
       "      <td>king and shaw</td>\n",
       "      <td>20.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>504</td>\n",
       "      <td>12:43:00</td>\n",
       "      <td>4128</td>\n",
       "      <td>2014-01-02 12:43:43</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>midday</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-02 14:01:01</th>\n",
       "      <td>Thursday</td>\n",
       "      <td>w</td>\n",
       "      <td>Held By</td>\n",
       "      <td>bingham and kingston road</td>\n",
       "      <td>13.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>501</td>\n",
       "      <td>14:01:00</td>\n",
       "      <td>4016</td>\n",
       "      <td>2014-01-02 14:01:01</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>midday</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-02 14:22:22</th>\n",
       "      <td>Thursday</td>\n",
       "      <td>w</td>\n",
       "      <td>Investigation</td>\n",
       "      <td>king st. and roncesvalles</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>504</td>\n",
       "      <td>14:22:00</td>\n",
       "      <td>4175</td>\n",
       "      <td>2014-01-02 14:22:22</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>midday</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-02 16:42:42</th>\n",
       "      <td>Thursday</td>\n",
       "      <td>e</td>\n",
       "      <td>Utilized Off Route</td>\n",
       "      <td>bathurst and king</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>504</td>\n",
       "      <td>16:42:00</td>\n",
       "      <td>4080</td>\n",
       "      <td>2014-01-02 16:42:42</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>aft_rush</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Day Direction             Incident  \\\n",
       "Report Date Time                                               \n",
       "2014-01-02 06:31:31  Thursday         e  Late Leaving Garage   \n",
       "2014-01-02 12:43:43  Thursday         e   Utilized Off Route   \n",
       "2014-01-02 14:01:01  Thursday         w              Held By   \n",
       "2014-01-02 14:22:22  Thursday         w        Investigation   \n",
       "2014-01-02 16:42:42  Thursday         e   Utilized Off Route   \n",
       "\n",
       "                                      Location  Min Delay  Min Gap  \\\n",
       "Report Date Time                                                     \n",
       "2014-01-02 06:31:31    dundas and roncesvalles        4.0      8.0   \n",
       "2014-01-02 12:43:43              king and shaw       20.0     22.0   \n",
       "2014-01-02 14:01:01  bingham and kingston road       13.0     19.0   \n",
       "2014-01-02 14:22:22  king st. and roncesvalles        7.0     11.0   \n",
       "2014-01-02 16:42:42          bathurst and king        3.0      6.0   \n",
       "\n",
       "                    Report Date Route      Time Vehicle    Report Date Time  \\\n",
       "Report Date Time                                                              \n",
       "2014-01-02 06:31:31  2014-01-02   505  06:31:00    4018 2014-01-02 06:31:31   \n",
       "2014-01-02 12:43:43  2014-01-02   504  12:43:00    4128 2014-01-02 12:43:43   \n",
       "2014-01-02 14:01:01  2014-01-02   501  14:01:00    4016 2014-01-02 14:01:01   \n",
       "2014-01-02 14:22:22  2014-01-02   504  14:22:00    4175 2014-01-02 14:22:22   \n",
       "2014-01-02 16:42:42  2014-01-02   504  16:42:00    4080 2014-01-02 16:42:42   \n",
       "\n",
       "                     year  month  daym  hour   time_of_day  target  \n",
       "Report Date Time                                                    \n",
       "2014-01-02 06:31:31  2014      1     2     6  morning_rush       0  \n",
       "2014-01-02 12:43:43  2014      1     2    12        midday       1  \n",
       "2014-01-02 14:01:01  2014      1     2    14        midday       1  \n",
       "2014-01-02 14:22:22  2014      1     2    14        midday       1  \n",
       "2014-01-02 16:42:42  2014      1     2    16      aft_rush       0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "merged_data after dropping extraneous columns\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Direction</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>Report Date</th>\n",
       "      <th>Route</th>\n",
       "      <th>hour</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Report Date Time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2014-01-02 06:31:31</th>\n",
       "      <td>e</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>505</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-02 12:43:43</th>\n",
       "      <td>e</td>\n",
       "      <td>20.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>504</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-02 14:01:01</th>\n",
       "      <td>w</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>501</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-02 14:22:22</th>\n",
       "      <td>w</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>504</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-02 16:42:42</th>\n",
       "      <td>e</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>504</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Direction  Min Delay Report Date Route  hour\n",
       "Report Date Time                                                \n",
       "2014-01-02 06:31:31         e        4.0  2014-01-02   505     6\n",
       "2014-01-02 12:43:43         e       20.0  2014-01-02   504    12\n",
       "2014-01-02 14:01:01         w       13.0  2014-01-02   501    14\n",
       "2014-01-02 14:22:22         w        7.0  2014-01-02   504    14\n",
       "2014-01-02 16:42:42         e        3.0  2014-01-02   504    16"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "result3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>count</th>\n",
       "      <th>Route</th>\n",
       "      <th>Direction</th>\n",
       "      <th>hour</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "      <th>day</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>0</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>1</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>2</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>3</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>e</td>\n",
       "      <td>4</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Report Date  count Route Direction  hour  year  month  daym  day  Min Delay  \\\n",
       "0  2014-01-01      0   301         e     0  2014      1     1    2        0.0   \n",
       "1  2014-01-01      0   301         e     1  2014      1     1    2        0.0   \n",
       "2  2014-01-01      0   301         e     2  2014      1     1    2        0.0   \n",
       "3  2014-01-01      0   301         e     3  2014      1     1    2        0.0   \n",
       "4  2014-01-01      0   301         e     4  2014      1     1    2        0.0   \n",
       "\n",
       "   target  \n",
       "0       0  \n",
       "1       0  \n",
       "2       0  \n",
       "3       0  \n",
       "4       0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of refactored dataset (2952599, 11)\n",
      "count of no delay  2892554\n",
      "count of delay  60045\n",
      "setting parameters for experiment  5\n",
      "early_stop is  True\n",
      "one_weight is  48.173103505704056\n",
      "epochs is  50\n",
      "es_monitor is  val_accuracy\n",
      "es_mode is  max\n"
     ]
    }
   ],
   "source": [
    "# master calls\n",
    "# get the path for data files\n",
    "path = get_path()\n",
    "print(\"path is\",path)\n",
    "# load route direction and delay data datframes\n",
    "directions_df, merged_data = ingest_data(path)\n",
    "merged_data = prep_merged_data(merged_data)\n",
    "print(\"shape of pre refactored dataset\", merged_data.shape)\n",
    "merged_data['year'].value_counts()\n",
    "merged_data.groupby(['Route','Direction']).size().reset_index().rename(columns={0:'count'}).tail(50)\n",
    "# create refactored dataframe with one row for each route / direction / timeslot combination\n",
    "merged_data = prep_sparse_df(directions_df, merged_data)\n",
    "print(\"shape of refactored dataset\", merged_data.shape)\n",
    "count_no_delay = merged_data[merged_data['target']==0].shape[0]\n",
    "count_delay = merged_data[merged_data['target']==1].shape[0]\n",
    "print(\"count of no delay \",count_no_delay)\n",
    "print(\"count of delay \",count_delay)\n",
    "# define parameters for the current experiment\n",
    "experiment_number = current_experiment\n",
    "early_stop, one_weight, epochs,es_monitor,es_mode = set_experiment_parameters(experiment_number, count_no_delay, count_delay)\n",
    "print(\"early_stop is \",early_stop)\n",
    "print(\"one_weight is \",one_weight)\n",
    "print(\"epochs is \",epochs)\n",
    "print(\"es_monitor is \",es_monitor)\n",
    "print(\"es_mode is \",es_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2952599, 11)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define test / training sets; encode categorical values; process text field\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get training and test data set\n",
    "def get_train_validation_test(dataset):\n",
    "    train, test = train_test_split(dataset, test_size = testproportion)\n",
    "    dtrain, dvalid = train_test_split(train, random_state=123, train_size=trainproportion)\n",
    "    print(\"Through train test split. Test proportion:\")\n",
    "    print(testproportion)\n",
    "    return(dtrain,dvalid,test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define feature categories <a name='definecategories' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all cols ['Report Date', 'count', 'Route', 'Direction', 'hour', 'year', 'month', 'daym', 'day', 'Min Delay', 'target']\n"
     ]
    }
   ],
   "source": [
    "allcols = list(merged_data)\n",
    "print(\"all cols\",allcols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the required column lists\n",
    "def def_col_lists():\n",
    "    textcols = [] # columns to deal with as text - replace entries with multiple IDs and use embeddings, RNN\n",
    "    continuouscols = [] # columns to deal with as continuous values - no embeddings\n",
    "    if targetcontinuous:\n",
    "        excludefromcolist = ['count','Report Date', 'target','count_md','Min Delay'] # columns to exclude completely from the model\n",
    "\n",
    "    else:\n",
    "        # if target column is not renamed Min Delay put Min Delay in exclusion list\n",
    "        excludefromcolist = ['count','Report Date', 'target','count_md', 'Min Delay'] # columns to exclude completely from the model\n",
    "    nontextcols = list(set(allcols) - set(textcols))\n",
    "    collist = list(set(nontextcols) - set(excludefromcolist) - set(continuouscols))\n",
    "    for col in continuouscols:\n",
    "        print(\"col is\",col)\n",
    "        merged_data[col] = merged_data[col].astype(float)\n",
    "        print(\"got through one\")\n",
    "        superset_data[col] = superset_data[col].astype(float)\n",
    "    # print column list lengths and contents:\n",
    "    print(\"allcols\",len(allcols))\n",
    "    print(\"excludefromcolist\",len(excludefromcolist))\n",
    "    print(excludefromcolist)\n",
    "    print(\"textcols\",len(textcols))\n",
    "    print(textcols)\n",
    "    print(\"continuouscols\",len(continuouscols))\n",
    "    print(continuouscols)\n",
    "    print(\"collist\",len(collist))\n",
    "    print(collist)\n",
    "    return(collist,continuouscols,textcols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Invoke Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define keras variables\n",
    "\n",
    "\n",
    "# X for the features used\n",
    "\n",
    "def get_keras_vars(dataset):\n",
    "    X = {}\n",
    "    dictlist = []\n",
    "    for col in collist:\n",
    "        if verboseout:\n",
    "            print(\"cat col is\",col)\n",
    "        X[col] = np.array(dataset[col])\n",
    "        dictlist.append(np.array(dataset[col]))\n",
    "\n",
    "    for col in textcols:\n",
    "        if verboseout:\n",
    "            print(\"text col is\",col)\n",
    "        X[col] = pad_sequences(dataset[col], maxlen=max_dict[col])\n",
    "        dictlist.append(pad_sequences(dataset[col], maxlen=max_dict[col]))\n",
    "\n",
    "    for col in continuouscols:\n",
    "        if verboseout:\n",
    "            print(\"cont col is\",col)\n",
    "        X[col] = np.array(dataset[col])\n",
    "        dictlist.append(np.array(dataset[col]))\n",
    "\n",
    "    return X, dictlist\n",
    "\n",
    "def get_keras_list_only(X_in):\n",
    "    dictlist = []\n",
    "    for key, value in X_in.items():\n",
    "        print(\"X def loop key\",key)\n",
    "        print(\"value shape\",value.shape)\n",
    "        temp = [key,value]\n",
    "        dictlist.append(value)\n",
    "    return dictlist\n",
    "\n",
    "def get_keras_np(X_in):\n",
    "    return np.array(list(X_in.items()),dtype=object)\n",
    "# np.array(list(result.items()), dtype=dtype)\n",
    "\n",
    "# the deployment API for Watson Studio can only take a list/array, not a dictionary, so define list-only version for input\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "allcols 11\n",
      "excludefromcolist 5\n",
      "['count', 'Report Date', 'target', 'count_md', 'Min Delay']\n",
      "textcols 0\n",
      "[]\n",
      "continuouscols 0\n",
      "[]\n",
      "collist 7\n",
      "['month', 'hour', 'daym', 'Route', 'day', 'Direction', 'year']\n",
      "fill empty xform\n",
      "col is  month\n",
      "col is  hour\n",
      "col is  daym\n",
      "col is  Route\n",
      "col is  day\n",
      "col is  Direction\n",
      "col is  year\n",
      "transform col is  month\n",
      "after transform col is  month\n",
      "transform col is  hour\n",
      "after transform col is  hour\n",
      "transform col is  daym\n",
      "after transform col is  daym\n",
      "transform col is  Route\n",
      "after transform col is  Route\n",
      "transform col is  day\n",
      "after transform col is  day\n",
      "transform col is  Direction\n",
      "after transform col is  Direction\n",
      "transform col is  year\n",
      "after transform col is  year\n",
      "Through train test split. Test proportion:\n",
      "0.2\n",
      "cat col is month\n",
      "cat col is hour\n",
      "cat col is daym\n",
      "cat col is Route\n",
      "cat col is day\n",
      "cat col is Direction\n",
      "cat col is year\n",
      "cat col is month\n",
      "cat col is hour\n",
      "cat col is daym\n",
      "cat col is Route\n",
      "cat col is day\n",
      "cat col is Direction\n",
      "cat col is year\n",
      "cat col is month\n",
      "cat col is hour\n",
      "cat col is daym\n",
      "cat col is Route\n",
      "cat col is day\n",
      "cat col is Direction\n",
      "cat col is year\n",
      "keras variables defined\n",
      "X_train_list [array([11,  4,  5, ...,  3,  9,  9], dtype=int64), array([15,  7, 12, ..., 20,  6, 23], dtype=int64), array([ 6,  0,  4, ..., 15, 19, 14], dtype=int64), array([ 9,  7, 12, ...,  9,  3,  8]), array([6, 3, 2, ..., 6, 5, 0], dtype=int64), array([4, 3, 4, ..., 4, 3, 4]), array([0, 0, 5, ..., 3, 4, 4], dtype=int64)]\n"
     ]
    }
   ],
   "source": [
    "# master block to invoke pipeline\n",
    "\n",
    "# build fully qualified names for the files for saving the pipelines\n",
    "pipeline_path = get_pipeline_path()\n",
    "pipeline1_file_name = os.path.join(pipeline_path,'sc_delay_pipleline'+modifier+'.pkl')\n",
    "pipeline2_file_name = os.path.join(pipeline_path,'sc_delay_pipleline_keras_prep'+modifier+'.pkl')\n",
    "\n",
    "# define column lists:\n",
    "collist,continuouscols,textcols = def_col_lists()\n",
    "\n",
    "# create objects of the pipeline classes\n",
    "fe = fill_empty()\n",
    "ec = encode_categorical()\n",
    "pk = prep_for_keras_input()\n",
    "\n",
    "# need to implement the pipeline in two parts:\n",
    "# 1. fill empty + encode categoricals\n",
    "# 2. prep for Keras\n",
    "# because part 1 needs to be applied to the entire dataset and part 2 to the individual train, validate, and test sets\n",
    "\n",
    "\n",
    "sc_delay_pipeline = Pipeline([('fill_empty',fe),('encode_categorical',ec)])\n",
    "sc_delay_pipeline_keras_prep = Pipeline([('prep_for_keras',pk)])\n",
    "\n",
    "\n",
    "\n",
    "# provide the value for each parameter of each of the pipeline classes\n",
    "\n",
    "sc_delay_pipeline.set_params(fill_empty__collist = collist, fill_empty__continuouscols = continuouscols,\n",
    "                            fill_empty__textcols = textcols,encode_categorical__col_list = collist)\n",
    "sc_delay_pipeline_keras_prep.set_params(prep_for_keras__collist = collist,\n",
    "                            prep_for_keras__continuouscols = continuouscols,\n",
    "                            prep_for_keras__textcols = textcols)\n",
    "\n",
    "# fit the input dataset to the pipeline\n",
    "\n",
    "# first fit the first segment of pipeline on the whole dataset\n",
    "X = sc_delay_pipeline.fit_transform(merged_data)\n",
    "max_dict = ec.max_dict\n",
    "# then split dataset\n",
    "dump(sc_delay_pipeline, open(pipeline1_file_name,'wb'))\n",
    "dump(sc_delay_pipeline_keras_prep, open(pipeline2_file_name,'wb'))\n",
    "dtrain, dvalid, test = get_train_validation_test(X)\n",
    "# then apply second portion of pipeline to each subset\n",
    "\n",
    "X_train, X_train_list = get_keras_vars(dtrain)\n",
    "X_valid, X_valid_list = get_keras_vars(dvalid)\n",
    "X_test,X_test_list = get_keras_vars(test)\n",
    "\n",
    "print(\"keras variables defined\")\n",
    "print(\"X_train_list\",X_train_list)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define and fit model <a name='modelfit' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "about to define embeddings\n",
      "textmax is 50\n",
      "through loops for cols\n",
      "through definition of non-text parts of main_l\n",
      "main_l Tensor(\"concatenate_5/Identity:0\", shape=(None, 70), dtype=float32)\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "month (InputLayer)              [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "hour (InputLayer)               [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 1, 10)        120         month[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 1, 10)        240         hour[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "daym (InputLayer)               [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 1, 10)        40          embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 1, 10)        40          embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 1, 10)        310         daym[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "Route (InputLayer)              [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 10)           0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 10)           0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 1, 10)        40          embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, 1, 10)        140         Route[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "day (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 10)           0           flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 10)           0           flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 10)           0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 1, 10)        40          embedding_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)         (None, 1, 10)        70          day[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "Direction (InputLayer)          [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 20)           0           dropout[0][0]                    \n",
      "                                                                 dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 10)           0           flatten_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 10)           0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 1, 10)        40          embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_5 (Embedding)         (None, 1, 10)        50          Direction[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "year (InputLayer)               [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 30)           0           concatenate[0][0]                \n",
      "                                                                 dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 10)           0           flatten_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 10)           0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 1, 10)        40          embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (None, 1, 10)        60          year[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 40)           0           concatenate_1[0][0]              \n",
      "                                                                 dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 10)           0           flatten_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 10)           0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 1, 10)        40          embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 50)           0           concatenate_2[0][0]              \n",
      "                                                                 dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 10)           0           flatten_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)             (None, 10)           0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 60)           0           concatenate_3[0][0]              \n",
      "                                                                 dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 10)           0           flatten_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 70)           0           concatenate_4[0][0]              \n",
      "                                                                 dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1)            71          concatenate_5[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 1,341\n",
      "Trainable params: 1,201\n",
      "Non-trainable params: 140\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# define model in Keras\n",
    "\n",
    "def get_model():\n",
    "\n",
    "\n",
    "    catinputs = {} # list of categorical inputs\n",
    "    textinputs = {} # list of text inputs\n",
    "    continputs = {} # list of continuous inputs\n",
    "    embeddings = {}\n",
    "    textembeddings = {}\n",
    "    catemb = 10 # size of categorical embeddings\n",
    "    textemb = 50 # size of text embeddings\n",
    "\n",
    "\n",
    "    print(\"about to define embeddings\")\n",
    "    collistfix = []\n",
    "    textlayerlist = []\n",
    "    inputlayerlist = []\n",
    "    i = 0\n",
    "    print(\"textmax is\",textmax)\n",
    "    # define layers for categorical columns\n",
    "    for col in collist:\n",
    "        catinputs[col] = Input(shape=[1],name=col)\n",
    "        inputlayerlist.append(catinputs[col])\n",
    "        #print(\"inputname\",inputname)\n",
    "        embeddings[col] = (Embedding(max_dict[col],catemb) (catinputs[col]))\n",
    "        # batchnorm all\n",
    "        embeddings[col] = (BatchNormalization() (embeddings[col]))\n",
    "        collistfix.append(embeddings[col])\n",
    "\n",
    "\n",
    "\n",
    "    # define layers for text columns\n",
    "    if includetext:\n",
    "        for col in textcols:\n",
    "            print(\"col\",col)\n",
    "            textinputs[col] = Input(shape=[X_train[col].shape[1]], name=col)\n",
    "            print(\"text input shape\",X_train[col].shape[1])\n",
    "            inputlayerlist.append(textinputs[col])\n",
    "            textembeddings[col] = (Embedding(textmax,textemb) (textinputs[col]))\n",
    "            textembeddings[col] = (BatchNormalization() (textembeddings[col]))\n",
    "            textembeddings[col] = Dropout(dropout_rate) ( GRU(16,kernel_regularizer=l2(l2_lambda)) (textembeddings[col]))\n",
    "            collistfix.append(textembeddings[col])\n",
    "            print(\"max in the midst\",np.max([np.max(train[col].max()), np.max(test[col].max())])+10)\n",
    "        print(\"through loops for cols\")\n",
    "\n",
    "    # define layers for continuous columns\n",
    "    for col in continuouscols:\n",
    "        continputs[col] = Input(shape=[1],name=col)\n",
    "        inputlayerlist.append(continputs[col])\n",
    "\n",
    "\n",
    "\n",
    "    # build up layers\n",
    "    # main_l = concatenate([Dropout(dropout_rate) (Flatten() (embeddings['Vehicle']) ),Dropout(dropout_rate) (Flatten() (embeddings['Direction']) )])\n",
    "    main_l = concatenate([Dropout(dropout_rate) (Flatten() (embeddings[collist[0]]) ),Dropout(dropout_rate) (Flatten() (embeddings[collist[1]]) )])\n",
    "    for cols in collist:\n",
    "        if (cols != collist[0]) & (cols != collist[1]):\n",
    "            main_l = concatenate([main_l,Dropout(dropout_rate) (Flatten() (embeddings[cols]) )])\n",
    "\n",
    "    print(\"through definition of non-text parts of main_l\")\n",
    "    if includetext:\n",
    "        for col in textcols:\n",
    "            main_l = concatenate([main_l,textembeddings[col]])\n",
    "\n",
    "    for col in continuouscols:\n",
    "        main_l = concatenate([main_l,continputs[col]])\n",
    "\n",
    "    print(\"main_l\", main_l)\n",
    "\n",
    "\n",
    "\n",
    "    # define output layer\n",
    "    output = Dense(1, activation=output_activation) (main_l)\n",
    "\n",
    "    # define model\n",
    "\n",
    "    model = Model(inputlayerlist, output)\n",
    "\n",
    "\n",
    "    # define optimizer\n",
    "    optimizer = SGD(lr=learning_rate)\n",
    "\n",
    "    # compile model\n",
    "    model.compile(loss=loss_func, optimizer=optimizer, metrics=[\"accuracy\"], weighted_metrics=[\"accuracy\"])\n",
    "    # model.compile(loss=loss_func, optimizer=optimizer, metrics=[\"accuracy\", Recall(), Precision()], weighted_metrics=[\"accuracy\"])\n",
    "    \n",
    "\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "model = get_model()\n",
    "\n",
    "# output model summary\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model plot\n",
    "if save_model_plot:\n",
    "    model_plot_file = \"model_plot\"+modifier+\".png\"\n",
    "    model_plot_path = os.path.join(get_path(),model_plot_file)\n",
    "    print(\"model plot path: \",model_plot_path)\n",
    "    plot_model(model, to_file=model_plot_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up early stopping\n",
    "def set_early_stop(es_monitor, es_mode):\n",
    "    ''' given monitoring parameter es_monitor and mode es_mode, define early stopping callback, save model callback, and \n",
    "    TensorBoard callback'''\n",
    "    # define callback for early stopping\n",
    "    callback_list = []\n",
    "    es = EarlyStopping(monitor=es_monitor, mode=es_mode, verbose=1,patience = patience_threshold)\n",
    "    callback_list.append(es)\n",
    "    model_path = get_model_path()\n",
    "    save_model_path = os.path.join(model_path,'scmodel'+modifier+\"_\"+str(experiment_number)+'.h5')\n",
    "    # define callback to save best model\n",
    "    mc = ModelCheckpoint(save_model_path, monitor=es_monitor, mode=es_mode, verbose=1, save_best_only=True)\n",
    "    callback_list.append(mc)\n",
    "    # define callback for TensorBoard\n",
    "    if tensorboard_callback:\n",
    "        tensorboard_log_dir = os.path.join(get_path(),\"tensorboard_log\",datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "        tensorboard = TensorBoard(log_dir= tensorboard_log_dir)\n",
    "        callback_list.append(tensorboard)\n",
    "    return(callback_list,save_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text cols []\n",
      "dropout  0.0003\n",
      "L2 lambda  0.0003\n",
      "batch size  1000\n",
      "epochs 50\n",
      "learning_rate 0.001\n",
      "loss function binary_crossentropy\n",
      "output activation function hard_sigmoid\n",
      "patience_threshold is  15\n",
      "experiment number is  5\n",
      "early stop is True\n",
      "about to define embeddings\n",
      "textmax is 50\n",
      "through loops for cols\n",
      "through definition of non-text parts of main_l\n",
      "main_l Tensor(\"concatenate_11/Identity:0\", shape=(None, 70), dtype=float32)\n",
      "Train on 1889663 samples, validate on 472416 samples\n",
      "Epoch 1/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 1.1708 - accuracy: 0.6072 - accuracy_1: 0.7342\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.49339, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 16s 8us/sample - loss: 1.1791 - accuracy: 0.6071 - accuracy_1: 0.7337 - val_loss: 2.5190 - val_accuracy: 0.4934 - val_accuracy_1: 0.4934\n",
      "Epoch 2/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 1.8978 - accuracy: 0.5637 - accuracy_1: 0.6705\n",
      "Epoch 00002: val_accuracy improved from 0.49339 to 0.63860, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 14s 7us/sample - loss: 1.8964 - accuracy: 0.5638 - accuracy_1: 0.6705 - val_loss: 0.6036 - val_accuracy: 0.6386 - val_accuracy_1: 0.6386\n",
      "Epoch 3/50\n",
      "1884000/1889663 [============================>.] - ETA: 0s - loss: 0.9137 - accuracy: 0.6831 - accuracy_1: 0.7647\n",
      "Epoch 00003: val_accuracy improved from 0.63860 to 0.70360, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 17s 9us/sample - loss: 0.9136 - accuracy: 0.6832 - accuracy_1: 0.7647 - val_loss: 0.5414 - val_accuracy: 0.7036 - val_accuracy_1: 0.7036\n",
      "Epoch 4/50\n",
      "1888000/1889663 [============================>.] - ETA: 0s - loss: 0.8850 - accuracy: 0.7141 - accuracy_1: 0.7709\n",
      "Epoch 00004: val_accuracy improved from 0.70360 to 0.72084, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 0.8849 - accuracy: 0.7141 - accuracy_1: 0.7709 - val_loss: 0.5020 - val_accuracy: 0.7208 - val_accuracy_1: 0.7208\n",
      "Epoch 5/50\n",
      "1885000/1889663 [============================>.] - ETA: 0s - loss: 0.9142 - accuracy: 0.7254 - accuracy_1: 0.7656\n",
      "Epoch 00005: val_accuracy improved from 0.72084 to 0.72935, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 0.9140 - accuracy: 0.7254 - accuracy_1: 0.7656 - val_loss: 0.4953 - val_accuracy: 0.7294 - val_accuracy_1: 0.7294\n",
      "Epoch 6/50\n",
      "1888000/1889663 [============================>.] - ETA: 0s - loss: 0.8754 - accuracy: 0.7269 - accuracy_1: 0.7731\n",
      "Epoch 00006: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 0.8754 - accuracy: 0.7269 - accuracy_1: 0.7731 - val_loss: 0.4836 - val_accuracy: 0.7225 - val_accuracy_1: 0.7225\n",
      "Epoch 7/50\n",
      "1882000/1889663 [============================>.] - ETA: 0s - loss: 0.8749 - accuracy: 0.7267 - accuracy_1: 0.7735\n",
      "Epoch 00007: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 14s 7us/sample - loss: 0.8751 - accuracy: 0.7267 - accuracy_1: 0.7735 - val_loss: 0.4717 - val_accuracy: 0.7281 - val_accuracy_1: 0.7281\n",
      "Epoch 8/50\n",
      "1888000/1889663 [============================>.] - ETA: 0s - loss: 1.0595 - accuracy: 0.7106 - accuracy_1: 0.7380\n",
      "Epoch 00008: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 1.0595 - accuracy: 0.7106 - accuracy_1: 0.7380 - val_loss: 0.5076 - val_accuracy: 0.6669 - val_accuracy_1: 0.6669\n",
      "Epoch 9/50\n",
      "1883000/1889663 [============================>.] - ETA: 0s - loss: 0.9657 - accuracy: 0.6826 - accuracy_1: 0.7497\n",
      "Epoch 00009: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 0.9656 - accuracy: 0.6828 - accuracy_1: 0.7497 - val_loss: 0.5287 - val_accuracy: 0.7234 - val_accuracy_1: 0.7234\n",
      "Epoch 10/50\n",
      "1883000/1889663 [============================>.] - ETA: 0s - loss: 0.8943 - accuracy: 0.7272 - accuracy_1: 0.7716\n",
      "Epoch 00010: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 14s 7us/sample - loss: 0.8943 - accuracy: 0.7271 - accuracy_1: 0.7715 - val_loss: 0.4978 - val_accuracy: 0.7251 - val_accuracy_1: 0.7251\n",
      "Epoch 11/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 0.8796 - accuracy: 0.7228 - accuracy_1: 0.7725\n",
      "Epoch 00011: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8799 - accuracy: 0.7228 - accuracy_1: 0.7724 - val_loss: 0.4959 - val_accuracy: 0.7168 - val_accuracy_1: 0.7168\n",
      "Epoch 12/50\n",
      "1883000/1889663 [============================>.] - ETA: 0s - loss: 0.8820 - accuracy: 0.7147 - accuracy_1: 0.7721\n",
      "Epoch 00012: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8819 - accuracy: 0.7147 - accuracy_1: 0.7721 - val_loss: 0.4898 - val_accuracy: 0.7127 - val_accuracy_1: 0.7127\n",
      "Epoch 13/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 1.0876 - accuracy: 0.7229 - accuracy_1: 0.7459\n",
      "Epoch 00013: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 1.0871 - accuracy: 0.7229 - accuracy_1: 0.7459 - val_loss: 0.4882 - val_accuracy: 0.7197 - val_accuracy_1: 0.7197\n",
      "Epoch 14/50\n",
      "1884000/1889663 [============================>.] - ETA: 0s - loss: 0.8976 - accuracy: 0.7226 - accuracy_1: 0.7674\n",
      "Epoch 00014: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 0.8974 - accuracy: 0.7226 - accuracy_1: 0.7673 - val_loss: 0.4801 - val_accuracy: 0.7236 - val_accuracy_1: 0.7236\n",
      "Epoch 15/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 0.9110 - accuracy: 0.7276 - accuracy_1: 0.7636\n",
      "Epoch 00015: val_accuracy did not improve from 0.72935\n",
      "1889663/1889663 [==============================] - 14s 7us/sample - loss: 0.9108 - accuracy: 0.7276 - accuracy_1: 0.7637 - val_loss: 0.4862 - val_accuracy: 0.7293 - val_accuracy_1: 0.7293\n",
      "Epoch 16/50\n",
      "1883000/1889663 [============================>.] - ETA: 0s - loss: 0.8740 - accuracy: 0.7293 - accuracy_1: 0.7708\n",
      "Epoch 00016: val_accuracy improved from 0.72935 to 0.72990, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 0.8741 - accuracy: 0.7293 - accuracy_1: 0.7707 - val_loss: 0.4791 - val_accuracy: 0.7299 - val_accuracy_1: 0.7299\n",
      "Epoch 17/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 0.8709 - accuracy: 0.7310 - accuracy_1: 0.7718\n",
      "Epoch 00017: val_accuracy improved from 0.72990 to 0.73048, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8709 - accuracy: 0.7310 - accuracy_1: 0.7718 - val_loss: 0.4734 - val_accuracy: 0.7305 - val_accuracy_1: 0.7305\n",
      "Epoch 18/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 0.8761 - accuracy: 0.7313 - accuracy_1: 0.7733\n",
      "Epoch 00018: val_accuracy did not improve from 0.73048\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8760 - accuracy: 0.7312 - accuracy_1: 0.7732 - val_loss: 0.4773 - val_accuracy: 0.7231 - val_accuracy_1: 0.7231\n",
      "Epoch 19/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 0.8710 - accuracy: 0.7257 - accuracy_1: 0.7741\n",
      "Epoch 00019: val_accuracy did not improve from 0.73048\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8709 - accuracy: 0.7258 - accuracy_1: 0.7741 - val_loss: 0.4759 - val_accuracy: 0.7291 - val_accuracy_1: 0.7291\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20/50\n",
      "1885000/1889663 [============================>.] - ETA: 0s - loss: 0.8715 - accuracy: 0.7305 - accuracy_1: 0.7734\n",
      "Epoch 00020: val_accuracy did not improve from 0.73048\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8713 - accuracy: 0.7305 - accuracy_1: 0.7734 - val_loss: 0.4781 - val_accuracy: 0.7291 - val_accuracy_1: 0.7291\n",
      "Epoch 21/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 0.8702 - accuracy: 0.7313 - accuracy_1: 0.7733\n",
      "Epoch 00021: val_accuracy improved from 0.73048 to 0.73232, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8702 - accuracy: 0.7313 - accuracy_1: 0.7733 - val_loss: 0.4696 - val_accuracy: 0.7323 - val_accuracy_1: 0.7323\n",
      "Epoch 22/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 1.0354 - accuracy: 0.6869 - accuracy_1: 0.7619\n",
      "Epoch 00022: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 1.0351 - accuracy: 0.6869 - accuracy_1: 0.7619 - val_loss: 0.4988 - val_accuracy: 0.6874 - val_accuracy_1: 0.6874\n",
      "Epoch 23/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 0.8851 - accuracy: 0.7017 - accuracy_1: 0.7720\n",
      "Epoch 00023: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8849 - accuracy: 0.7017 - accuracy_1: 0.7720 - val_loss: 0.4880 - val_accuracy: 0.7120 - val_accuracy_1: 0.7120\n",
      "Epoch 24/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 0.8876 - accuracy: 0.7250 - accuracy_1: 0.7717\n",
      "Epoch 00024: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8880 - accuracy: 0.7250 - accuracy_1: 0.7717 - val_loss: 0.4832 - val_accuracy: 0.7254 - val_accuracy_1: 0.7254\n",
      "Epoch 25/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 0.8726 - accuracy: 0.7277 - accuracy_1: 0.7725\n",
      "Epoch 00025: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 14s 7us/sample - loss: 0.8725 - accuracy: 0.7277 - accuracy_1: 0.7725 - val_loss: 0.4751 - val_accuracy: 0.7302 - val_accuracy_1: 0.7302\n",
      "Epoch 26/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 0.8692 - accuracy: 0.7307 - accuracy_1: 0.7732\n",
      "Epoch 00026: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 0.8692 - accuracy: 0.7307 - accuracy_1: 0.7732 - val_loss: 0.4702 - val_accuracy: 0.7300 - val_accuracy_1: 0.7300\n",
      "Epoch 27/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 0.8689 - accuracy: 0.7304 - accuracy_1: 0.7735\n",
      "Epoch 00027: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8688 - accuracy: 0.7304 - accuracy_1: 0.7735 - val_loss: 0.4703 - val_accuracy: 0.7319 - val_accuracy_1: 0.7319\n",
      "Epoch 28/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 0.8688 - accuracy: 0.7331 - accuracy_1: 0.7743\n",
      "Epoch 00028: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 0.8687 - accuracy: 0.7331 - accuracy_1: 0.7744 - val_loss: 0.4710 - val_accuracy: 0.7318 - val_accuracy_1: 0.7318\n",
      "Epoch 29/50\n",
      "1883000/1889663 [============================>.] - ETA: 0s - loss: 4.7266 - accuracy: 0.6866 - accuracy_1: 0.6860\n",
      "Epoch 00029: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 4.7492 - accuracy: 0.6862 - accuracy_1: 0.6856 - val_loss: 6.0403 - val_accuracy: 0.5383 - val_accuracy_1: 0.5383\n",
      "Epoch 30/50\n",
      "1884000/1889663 [============================>.] - ETA: 0s - loss: 9.9071 - accuracy: 0.5801 - accuracy_1: 0.6066\n",
      "Epoch 00030: val_accuracy did not improve from 0.73232\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 9.9030 - accuracy: 0.5804 - accuracy_1: 0.6067 - val_loss: 3.9905 - val_accuracy: 0.6959 - val_accuracy_1: 0.6959\n",
      "Epoch 31/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 8.3139 - accuracy: 0.7046 - accuracy_1: 0.6653\n",
      "Epoch 00031: val_accuracy improved from 0.73232 to 0.83433, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 8.3160 - accuracy: 0.7047 - accuracy_1: 0.6652 - val_loss: 2.2206 - val_accuracy: 0.8343 - val_accuracy_1: 0.8343\n",
      "Epoch 32/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 8.6429 - accuracy: 0.7712 - accuracy_1: 0.6629\n",
      "Epoch 00032: val_accuracy improved from 0.83433 to 0.86858, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 8.6434 - accuracy: 0.7714 - accuracy_1: 0.6629 - val_loss: 1.6415 - val_accuracy: 0.8686 - val_accuracy_1: 0.8686\n",
      "Epoch 33/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 8.6391 - accuracy: 0.7384 - accuracy_1: 0.6719\n",
      "Epoch 00033: val_accuracy did not improve from 0.86858\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 8.6395 - accuracy: 0.7384 - accuracy_1: 0.6719 - val_loss: 3.2428 - val_accuracy: 0.7473 - val_accuracy_1: 0.7473\n",
      "Epoch 34/50\n",
      "1888000/1889663 [============================>.] - ETA: 0s - loss: 8.2150 - accuracy: 0.8279 - accuracy_1: 0.6799\n",
      "Epoch 00034: val_accuracy did not improve from 0.86858\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 8.2141 - accuracy: 0.8279 - accuracy_1: 0.6799 - val_loss: 1.7206 - val_accuracy: 0.8548 - val_accuracy_1: 0.8548\n",
      "Epoch 35/50\n",
      "1888000/1889663 [============================>.] - ETA: 0s - loss: 7.0973 - accuracy: 0.8423 - accuracy_1: 0.7094\n",
      "Epoch 00035: val_accuracy did not improve from 0.86858\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 7.0964 - accuracy: 0.8423 - accuracy_1: 0.7094 - val_loss: 1.5611 - val_accuracy: 0.8599 - val_accuracy_1: 0.8599\n",
      "Epoch 36/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 5.2587 - accuracy: 0.8224 - accuracy_1: 0.7582\n",
      "Epoch 00036: val_accuracy did not improve from 0.86858\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 5.2594 - accuracy: 0.8224 - accuracy_1: 0.7582 - val_loss: 2.1567 - val_accuracy: 0.7930 - val_accuracy_1: 0.7930\n",
      "Epoch 37/50\n",
      "1885000/1889663 [============================>.] - ETA: 0s - loss: 4.8940 - accuracy: 0.8460 - accuracy_1: 0.7609\n",
      "Epoch 00037: val_accuracy improved from 0.86858 to 0.87181, saving model to C:\\personal\\manning\\deep_learning_for_structured_data\\models\\scmodelmay16_2020_5.h5\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 4.8938 - accuracy: 0.8461 - accuracy_1: 0.7608 - val_loss: 1.1973 - val_accuracy: 0.8718 - val_accuracy_1: 0.8718\n",
      "Epoch 38/50\n",
      "1886000/1889663 [============================>.] - ETA: 0s - loss: 3.5512 - accuracy: 0.8331 - accuracy_1: 0.7805\n",
      "Epoch 00038: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 3.5518 - accuracy: 0.8331 - accuracy_1: 0.7804 - val_loss: 1.2037 - val_accuracy: 0.8228 - val_accuracy_1: 0.8228\n",
      "Epoch 39/50\n",
      "1882000/1889663 [============================>.] - ETA: 0s - loss: 3.4131 - accuracy: 0.8258 - accuracy_1: 0.7949\n",
      "Epoch 00039: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 3.4126 - accuracy: 0.8259 - accuracy_1: 0.7949 - val_loss: 1.1794 - val_accuracy: 0.8255 - val_accuracy_1: 0.8255\n",
      "Epoch 40/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 3.3591 - accuracy: 0.8295 - accuracy_1: 0.7948\n",
      "Epoch 00040: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 3.3583 - accuracy: 0.8295 - accuracy_1: 0.7948 - val_loss: 0.7928 - val_accuracy: 0.8454 - val_accuracy_1: 0.8454\n",
      "Epoch 41/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 3.0932 - accuracy: 0.8267 - accuracy_1: 0.7973\n",
      "Epoch 00041: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 3.0937 - accuracy: 0.8267 - accuracy_1: 0.7973 - val_loss: 0.8036 - val_accuracy: 0.8214 - val_accuracy_1: 0.8214\n",
      "Epoch 42/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 3.0299 - accuracy: 0.8213 - accuracy_1: 0.7952\n",
      "Epoch 00042: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 3.0299 - accuracy: 0.8213 - accuracy_1: 0.7952 - val_loss: 0.7852 - val_accuracy: 0.8301 - val_accuracy_1: 0.8301\n",
      "Epoch 43/50\n",
      "1885000/1889663 [============================>.] - ETA: 0s - loss: 3.0218 - accuracy: 0.8225 - accuracy_1: 0.7978\n",
      "Epoch 00043: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 3.0206 - accuracy: 0.8224 - accuracy_1: 0.7978 - val_loss: 0.8136 - val_accuracy: 0.8199 - val_accuracy_1: 0.8199\n",
      "Epoch 44/50\n",
      "1888000/1889663 [============================>.] - ETA: 0s - loss: 3.0105 - accuracy: 0.8223 - accuracy_1: 0.7981\n",
      "Epoch 00044: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 3.0119 - accuracy: 0.8223 - accuracy_1: 0.7980 - val_loss: 0.7943 - val_accuracy: 0.8285 - val_accuracy_1: 0.8285\n",
      "Epoch 45/50\n",
      "1882000/1889663 [============================>.] - ETA: 0s - loss: 3.0083 - accuracy: 0.8228 - accuracy_1: 0.7989\n",
      "Epoch 00045: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 3.0076 - accuracy: 0.8229 - accuracy_1: 0.7989 - val_loss: 0.7777 - val_accuracy: 0.8363 - val_accuracy_1: 0.8363\n",
      "Epoch 46/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 3.0054 - accuracy: 0.8235 - accuracy_1: 0.7991\n",
      "Epoch 00046: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 3.0047 - accuracy: 0.8235 - accuracy_1: 0.7991 - val_loss: 0.7966 - val_accuracy: 0.8291 - val_accuracy_1: 0.8291\n",
      "Epoch 47/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 3.0035 - accuracy: 0.8241 - accuracy_1: 0.7990\n",
      "Epoch 00047: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 15s 8us/sample - loss: 3.0029 - accuracy: 0.8241 - accuracy_1: 0.7990 - val_loss: 0.8102 - val_accuracy: 0.8236 - val_accuracy_1: 0.8236\n",
      "Epoch 48/50\n",
      "1887000/1889663 [============================>.] - ETA: 0s - loss: 3.0003 - accuracy: 0.8240 - accuracy_1: 0.7993\n",
      "Epoch 00048: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 3.0010 - accuracy: 0.8240 - accuracy_1: 0.7993 - val_loss: 0.8104 - val_accuracy: 0.8242 - val_accuracy_1: 0.8242\n",
      "Epoch 49/50\n",
      "1883000/1889663 [============================>.] - ETA: 0s - loss: 3.1376 - accuracy: 0.8082 - accuracy_1: 0.7806\n",
      "Epoch 00049: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 3.1383 - accuracy: 0.8081 - accuracy_1: 0.7805 - val_loss: 0.8949 - val_accuracy: 0.7889 - val_accuracy_1: 0.7889\n",
      "Epoch 50/50\n",
      "1889000/1889663 [============================>.] - ETA: 0s - loss: 2.8410 - accuracy: 0.8008 - accuracy_1: 0.7946\n",
      "Epoch 00050: val_accuracy did not improve from 0.87181\n",
      "1889663/1889663 [==============================] - 14s 8us/sample - loss: 2.8411 - accuracy: 0.8008 - accuracy_1: 0.7946 - val_loss: 0.8560 - val_accuracy: 0.8073 - val_accuracy_1: 0.8073\n",
      "Wall time: 12min 12s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(\"text cols\",textcols)\n",
    "print(\"dropout \",dropout_rate)\n",
    "print(\"L2 lambda \",l2_lambda)\n",
    "print(\"batch size \",batch_size)\n",
    "print(\"epochs\",epochs)\n",
    "print(\"learning_rate\",learning_rate)\n",
    "print(\"loss function\",loss_func)\n",
    "print(\"output activation function\",output_activation)\n",
    "print(\"patience_threshold is \",patience_threshold)\n",
    "print(\"experiment number is \",experiment_number)\n",
    "print(\"early stop is\",early_stop)\n",
    "# get definitions\n",
    "callback_list, save_model_path = set_early_stop(es_monitor, es_mode)\n",
    "model = get_model()\n",
    "if early_stop:\n",
    "       modelfit = model.fit(X_train_list, dtrain.target, epochs=epochs, batch_size=batch_size\n",
    "        , validation_data=(X_valid_list, dvalid.target), class_weight = {0 : zero_weight, 1: one_weight}, verbose=1,callbacks=callback_list)\n",
    "else:\n",
    "    modelfit = model.fit(X_train_list, dtrain.target, epochs=epochs, batch_size=batch_size\n",
    "         , validation_data=(X_valid_list, dvalid.target), class_weight = {0 : zero_weight, 1: one_weight}, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model elements explicitly if not saved as part of early_stop\n",
    "if early_stop == False:\n",
    "    model_json = model.to_json()\n",
    "    model_path = get_model_path()\n",
    "    with open(os.path.join(model_path,'model'+modifier+'.json'), \"w\") as json_file:\n",
    "        json_file.write(model_json)\n",
    "    # serialize weights to HDF5\n",
    "    model.save_weights(os.path.join(model_path,'scweights'+modifier+'.h5'))\n",
    "    save_model_path = os.path.join(model_path,'scmodel'+modifier+'.h5')\n",
    "    model.save(save_model_path,save_format='h5')\n",
    "    # no early stop, so make current model saved_model\n",
    "    saved_model = model\n",
    "    print(\"Saved model, weights to disk\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics names  ['loss', 'accuracy', 'accuracy_1']\n",
      "Train: 0.873, Test: 0.873\n"
     ]
    }
   ],
   "source": [
    "saved_model = load_model(save_model_path)\n",
    "print(\"metrics names \",saved_model.metrics_names)\n",
    "# getting test and train accuracy is time consuming - check to see if experiment requires it\n",
    "if get_test_train_acc:\n",
    "    if len(saved_model.metrics_names) == 2:\n",
    "        # saved_model.evaluate returns ['loss', 'acc']\n",
    "        _, train_acc = saved_model.evaluate(X_train_list, dtrain.target, verbose=0)\n",
    "        _, test_acc = saved_model.evaluate(X_test, test.target, verbose=0)\n",
    "    else:\n",
    "        # saved_model.evaluate returns ['loss', 'accuracy', 'accuracy_1']\n",
    "        _, train_acc,_ = saved_model.evaluate(X_train_list, dtrain.target, verbose=0)\n",
    "        _, test_acc,_ = saved_model.evaluate(X_test, test.target, verbose=0)\n",
    "    print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([11,  4,  5, ...,  3,  9,  9], dtype=int64),\n",
       " array([15,  7, 12, ..., 20,  6, 23], dtype=int64),\n",
       " array([ 6,  0,  4, ..., 15, 19, 14], dtype=int64),\n",
       " array([ 9,  7, 12, ...,  9,  3,  8]),\n",
       " array([6, 3, 2, ..., 6, 5, 0], dtype=int64),\n",
       " array([4, 3, 4, ..., 4, 3, 4]),\n",
       " array([0, 0, 5, ..., 3, 4, 4], dtype=int64)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11,  4,  5, ...,  3,  9,  9], dtype=int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_list[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load saved model and weights <a name='reload' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "month (InputLayer)              [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "hour (InputLayer)               [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_7 (Embedding)         (None, 1, 10)        120         month[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "embedding_8 (Embedding)         (None, 1, 10)        240         hour[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "daym (InputLayer)               [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 1, 10)        40          embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 1, 10)        40          embedding_8[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_9 (Embedding)         (None, 1, 10)        310         daym[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "Route (InputLayer)              [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_7 (Flatten)             (None, 10)           0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_8 (Flatten)             (None, 10)           0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 1, 10)        40          embedding_9[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_10 (Embedding)        (None, 1, 10)        140         Route[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "day (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 10)           0           flatten_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, 10)           0           flatten_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_9 (Flatten)             (None, 10)           0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 1, 10)        40          embedding_10[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_11 (Embedding)        (None, 1, 10)        70          day[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "Direction (InputLayer)          [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)     (None, 20)           0           dropout_7[0][0]                  \n",
      "                                                                 dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 10)           0           flatten_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_10 (Flatten)            (None, 10)           0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 1, 10)        40          embedding_11[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_12 (Embedding)        (None, 1, 10)        50          Direction[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "year (InputLayer)               [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_7 (Concatenate)     (None, 30)           0           concatenate_6[0][0]              \n",
      "                                                                 dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 10)           0           flatten_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_11 (Flatten)            (None, 10)           0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 1, 10)        40          embedding_12[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_13 (Embedding)        (None, 1, 10)        60          year[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_8 (Concatenate)     (None, 40)           0           concatenate_7[0][0]              \n",
      "                                                                 dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, 10)           0           flatten_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_12 (Flatten)            (None, 10)           0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 1, 10)        40          embedding_13[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_9 (Concatenate)     (None, 50)           0           concatenate_8[0][0]              \n",
      "                                                                 dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)            (None, 10)           0           flatten_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_13 (Flatten)            (None, 10)           0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_10 (Concatenate)    (None, 60)           0           concatenate_9[0][0]              \n",
      "                                                                 dropout_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, 10)           0           flatten_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_11 (Concatenate)    (None, 70)           0           concatenate_10[0][0]             \n",
      "                                                                 dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            71          concatenate_11[0][0]             \n",
      "==================================================================================================\n",
      "Total params: 1,341\n",
      "Trainable params: 1,201\n",
      "Non-trainable params: 140\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "saved_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "if presaved == True:\n",
    "    batch_size = 1000\n",
    "    epochs = 1\n",
    "    modelfit2 = loaded_model.fit(X_train_list, dtrain.target, epochs=epochs, batch_size=batch_size\n",
    "         , validation_data=(X_valid_list, dvalid.target), verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predictions and renderings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions to parse and manipulate dates in the style of the input CSVs\n",
    "\n",
    "def create_date(year,month):\n",
    "    outdate = datetime.date(year,month,15)\n",
    "    return(outdate)\n",
    "\n",
    "def parse_bic_date(bic_date_in):\n",
    "    year = int(bic_date_in[0:4])\n",
    "    month = int(bic_date_in[-2:])\n",
    "    return(year,month)\n",
    "\n",
    "def create_date_from_bic(bic_date_in):\n",
    "    yr,mth = parse_bic_date(bic_date_in)\n",
    "    retdate = create_date(yr,mth)\n",
    "    return retdate\n",
    "\n",
    "def get_datecomp_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    # month_number = datetime.datetime.strptime(month_name, '%b').month\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b').month\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    # year = int(csv_date[-2:])\n",
    "    return (year,month,day)\n",
    "\n",
    "def get_date_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    # month_number = datetime.datetime.strptime(month_name, '%b').month\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b',coerce=True).month\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    # year = int(csv_date[-2:])\n",
    "    return (date(year,month,day))\n",
    "\n",
    "def get_year_from_csvdate (csv_date):\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    return (year)\n",
    "\n",
    "def get_month_from_csvdate (csv_date):\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b',coerce=True).month\n",
    "    return (month)\n",
    "\n",
    "def get_day_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    return (day)\n",
    "\n",
    "def get_weekday (date):\n",
    "    return(date.weekday())\n",
    "\n",
    "# pd.to_datetime(x, coerce=True)\n",
    "\n",
    "def validatedate(csv_text):\n",
    "    try:\n",
    "        datetime.datetime.strptime(csv_date[3:6], '%b')\n",
    "    except ValueError:\n",
    "        raise ValueError(\"Incorrect data format, should be YYYY-MM-DD\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[10,\n",
       " 8,\n",
       " 0,\n",
       " 7,\n",
       " 8,\n",
       " 10,\n",
       " 3,\n",
       " 7,\n",
       " 2,\n",
       " 11,\n",
       " 2,\n",
       " 1,\n",
       " 3,\n",
       " 8,\n",
       " 8,\n",
       " 6,\n",
       " 4,\n",
       " 9,\n",
       " 6,\n",
       " 8,\n",
       " 3,\n",
       " 11,\n",
       " 3,\n",
       " 10,\n",
       " 0,\n",
       " 11,\n",
       " 0,\n",
       " 6,\n",
       " 9,\n",
       " 1,\n",
       " 1,\n",
       " 11,\n",
       " 5,\n",
       " 9,\n",
       " 3,\n",
       " 2,\n",
       " 8,\n",
       " 0,\n",
       " 7,\n",
       " 7,\n",
       " 0,\n",
       " 2,\n",
       " 7,\n",
       " 6,\n",
       " 10,\n",
       " 2,\n",
       " 7,\n",
       " 2,\n",
       " 7,\n",
       " 8,\n",
       " 1,\n",
       " 4,\n",
       " 9,\n",
       " 8,\n",
       " 5,\n",
       " 1,\n",
       " 10,\n",
       " 5,\n",
       " 6,\n",
       " 10,\n",
       " 0,\n",
       " 11,\n",
       " 3,\n",
       " 8,\n",
       " 7,\n",
       " 8,\n",
       " 7,\n",
       " 5,\n",
       " 6,\n",
       " 7,\n",
       " 10,\n",
       " 10,\n",
       " 0,\n",
       " 9,\n",
       " 0,\n",
       " 10,\n",
       " 5,\n",
       " 1,\n",
       " 11,\n",
       " 2,\n",
       " 2,\n",
       " 6,\n",
       " 8,\n",
       " 6,\n",
       " 6,\n",
       " 2,\n",
       " 7,\n",
       " 6,\n",
       " 10,\n",
       " 11,\n",
       " 4,\n",
       " 9,\n",
       " 4,\n",
       " 0,\n",
       " 3,\n",
       " 10,\n",
       " 8,\n",
       " 5,\n",
       " 3,\n",
       " 11,\n",
       " 11,\n",
       " 3,\n",
       " 7,\n",
       " 0,\n",
       " 8,\n",
       " 6,\n",
       " 6,\n",
       " 11,\n",
       " 0,\n",
       " 3,\n",
       " 7,\n",
       " 5,\n",
       " 4,\n",
       " 10,\n",
       " 0,\n",
       " 6,\n",
       " 9,\n",
       " 11,\n",
       " 11,\n",
       " 10,\n",
       " 10,\n",
       " 3,\n",
       " 5,\n",
       " 11,\n",
       " 2,\n",
       " 3,\n",
       " 7,\n",
       " 8,\n",
       " 7,\n",
       " 0,\n",
       " 3,\n",
       " 1,\n",
       " 7,\n",
       " 10,\n",
       " 10,\n",
       " 3,\n",
       " 2,\n",
       " 6,\n",
       " 1,\n",
       " 3,\n",
       " 5,\n",
       " 11,\n",
       " 2,\n",
       " 1,\n",
       " 11,\n",
       " 11,\n",
       " 9,\n",
       " 6,\n",
       " 11,\n",
       " 2,\n",
       " 9,\n",
       " 10,\n",
       " 10,\n",
       " 10,\n",
       " 6,\n",
       " 8,\n",
       " 3,\n",
       " 5,\n",
       " 5,\n",
       " 5,\n",
       " 6,\n",
       " 4,\n",
       " 0,\n",
       " 0,\n",
       " 9,\n",
       " 3,\n",
       " 7,\n",
       " 11,\n",
       " 4,\n",
       " 2,\n",
       " 4,\n",
       " 5,\n",
       " 7,\n",
       " 0,\n",
       " 0,\n",
       " 4,\n",
       " 10,\n",
       " 4,\n",
       " 4,\n",
       " 0,\n",
       " 0,\n",
       " 7,\n",
       " 9,\n",
       " 10,\n",
       " 1,\n",
       " 11,\n",
       " 9,\n",
       " 3,\n",
       " 3,\n",
       " 1,\n",
       " 5,\n",
       " 2,\n",
       " 11,\n",
       " 6,\n",
       " 3,\n",
       " 2,\n",
       " 11,\n",
       " 2,\n",
       " 0,\n",
       " 6,\n",
       " 6,\n",
       " 9,\n",
       " 9,\n",
       " 3,\n",
       " 4,\n",
       " 6,\n",
       " 0,\n",
       " 7,\n",
       " 3,\n",
       " 9,\n",
       " 4,\n",
       " 9,\n",
       " 5,\n",
       " 11,\n",
       " 1,\n",
       " 7,\n",
       " 1,\n",
       " 11,\n",
       " 4,\n",
       " 3,\n",
       " 6,\n",
       " 3,\n",
       " 4,\n",
       " 6,\n",
       " 5,\n",
       " 1,\n",
       " 2,\n",
       " 2,\n",
       " 5,\n",
       " 2,\n",
       " 8,\n",
       " 2,\n",
       " 4,\n",
       " 11,\n",
       " 9,\n",
       " 7,\n",
       " 4,\n",
       " 5,\n",
       " 11,\n",
       " 11,\n",
       " 8,\n",
       " 7,\n",
       " 9,\n",
       " 1,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 3,\n",
       " 7,\n",
       " 2,\n",
       " 8,\n",
       " 11,\n",
       " 2,\n",
       " 0,\n",
       " 5,\n",
       " 2,\n",
       " 5,\n",
       " 2,\n",
       " 11,\n",
       " 9,\n",
       " 10,\n",
       " 3,\n",
       " 6,\n",
       " 7,\n",
       " 9,\n",
       " 11,\n",
       " 4,\n",
       " 11,\n",
       " 1,\n",
       " 0,\n",
       " 11,\n",
       " 3,\n",
       " 6,\n",
       " 1,\n",
       " 6,\n",
       " 4,\n",
       " 1,\n",
       " 7,\n",
       " 6,\n",
       " 8,\n",
       " 0,\n",
       " 3,\n",
       " 4,\n",
       " 4,\n",
       " 11,\n",
       " 6,\n",
       " 8,\n",
       " 10,\n",
       " 11,\n",
       " 2,\n",
       " 10,\n",
       " 4,\n",
       " 5,\n",
       " 10,\n",
       " 2,\n",
       " 10,\n",
       " 0,\n",
       " 7,\n",
       " 10,\n",
       " 0,\n",
       " 0,\n",
       " 9,\n",
       " 7,\n",
       " 6,\n",
       " 9,\n",
       " 2,\n",
       " 2,\n",
       " 5,\n",
       " 2,\n",
       " 2,\n",
       " 1,\n",
       " 7,\n",
       " 4,\n",
       " 1,\n",
       " 10,\n",
       " 0,\n",
       " 2,\n",
       " 7,\n",
       " 5,\n",
       " 8,\n",
       " 6,\n",
       " 2,\n",
       " 5,\n",
       " 7,\n",
       " 2,\n",
       " 9,\n",
       " 9,\n",
       " 10,\n",
       " 9,\n",
       " 10,\n",
       " 6,\n",
       " 9,\n",
       " 9,\n",
       " 8,\n",
       " 10,\n",
       " 5,\n",
       " 2,\n",
       " 0,\n",
       " 9,\n",
       " 0,\n",
       " 5,\n",
       " 4,\n",
       " 5,\n",
       " 1,\n",
       " 7,\n",
       " 2,\n",
       " 1,\n",
       " 4,\n",
       " 8,\n",
       " 2,\n",
       " 7,\n",
       " 11,\n",
       " 6,\n",
       " 2,\n",
       " 5,\n",
       " 6,\n",
       " 10,\n",
       " 7,\n",
       " 3,\n",
       " 5,\n",
       " 9,\n",
       " 9,\n",
       " 5,\n",
       " 8,\n",
       " 0,\n",
       " 8,\n",
       " 7,\n",
       " 8,\n",
       " 11,\n",
       " 9,\n",
       " 10,\n",
       " 1,\n",
       " 4,\n",
       " 4,\n",
       " 11,\n",
       " 0,\n",
       " 3,\n",
       " 2,\n",
       " 2,\n",
       " 2,\n",
       " 3,\n",
       " 6,\n",
       " 5,\n",
       " 4,\n",
       " 0,\n",
       " 1,\n",
       " 11,\n",
       " 3,\n",
       " 1,\n",
       " 7,\n",
       " 3,\n",
       " 9,\n",
       " 7,\n",
       " 7,\n",
       " 8,\n",
       " 6,\n",
       " 7,\n",
       " 11,\n",
       " 10,\n",
       " 3,\n",
       " 0,\n",
       " 2,\n",
       " 9,\n",
       " 4,\n",
       " 9,\n",
       " 1,\n",
       " 5,\n",
       " 7,\n",
       " 11,\n",
       " 11,\n",
       " 9,\n",
       " 3,\n",
       " 6,\n",
       " 6,\n",
       " 7,\n",
       " 4,\n",
       " 0,\n",
       " 9,\n",
       " 10,\n",
       " 8,\n",
       " 2,\n",
       " 11,\n",
       " 4,\n",
       " 4,\n",
       " 3,\n",
       " 6,\n",
       " 2,\n",
       " 4,\n",
       " 7,\n",
       " 1,\n",
       " 9,\n",
       " 5,\n",
       " 10,\n",
       " 5,\n",
       " 2,\n",
       " 1,\n",
       " 4,\n",
       " 10,\n",
       " 5,\n",
       " 1,\n",
       " 4,\n",
       " 2,\n",
       " 2,\n",
       " 4,\n",
       " 6,\n",
       " 4,\n",
       " 10,\n",
       " 2,\n",
       " 9,\n",
       " 2,\n",
       " 9,\n",
       " 6,\n",
       " 0,\n",
       " 8,\n",
       " 4,\n",
       " 0,\n",
       " 7,\n",
       " 3,\n",
       " 8,\n",
       " 6,\n",
       " 2,\n",
       " 5,\n",
       " 4,\n",
       " 7,\n",
       " 0,\n",
       " 11,\n",
       " 11,\n",
       " 5,\n",
       " 8,\n",
       " 5,\n",
       " 5,\n",
       " 5,\n",
       " 5,\n",
       " 9,\n",
       " 11,\n",
       " 1,\n",
       " 4,\n",
       " 2,\n",
       " 6,\n",
       " 2,\n",
       " 10,\n",
       " 8,\n",
       " 2,\n",
       " 10,\n",
       " 11,\n",
       " 6,\n",
       " 10,\n",
       " 3,\n",
       " 5,\n",
       " 2,\n",
       " 10,\n",
       " 4,\n",
       " 10,\n",
       " 0,\n",
       " 7,\n",
       " 9,\n",
       " 0,\n",
       " 1,\n",
       " 4,\n",
       " 4,\n",
       " 0,\n",
       " 10,\n",
       " 8,\n",
       " 7,\n",
       " 2,\n",
       " 9,\n",
       " 1,\n",
       " 8,\n",
       " 1,\n",
       " 10,\n",
       " 8,\n",
       " 2,\n",
       " 8,\n",
       " 7,\n",
       " 1,\n",
       " 8,\n",
       " 7,\n",
       " 0,\n",
       " 2,\n",
       " 3,\n",
       " 7,\n",
       " 8,\n",
       " 3,\n",
       " 5,\n",
       " 9,\n",
       " 6,\n",
       " 1,\n",
       " 8,\n",
       " 11,\n",
       " 5,\n",
       " 7,\n",
       " 4,\n",
       " 11,\n",
       " 0,\n",
       " 3,\n",
       " 9,\n",
       " 2,\n",
       " 4,\n",
       " 3,\n",
       " 1,\n",
       " 1,\n",
       " 11,\n",
       " 3,\n",
       " 3,\n",
       " 10,\n",
       " 5,\n",
       " 6,\n",
       " 0,\n",
       " 7,\n",
       " 9,\n",
       " 0,\n",
       " 9,\n",
       " 4,\n",
       " 2,\n",
       " 4,\n",
       " 5,\n",
       " 10,\n",
       " 1,\n",
       " 7,\n",
       " 5,\n",
       " 1,\n",
       " 0,\n",
       " 5,\n",
       " 6,\n",
       " 8,\n",
       " 5,\n",
       " 0,\n",
       " 4,\n",
       " 4,\n",
       " 7,\n",
       " 5,\n",
       " 6,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 7,\n",
       " 9,\n",
       " 6,\n",
       " 7,\n",
       " 9,\n",
       " 8,\n",
       " 6,\n",
       " 10,\n",
       " 5,\n",
       " 4,\n",
       " 9,\n",
       " 9,\n",
       " 2,\n",
       " 0,\n",
       " 4,\n",
       " 3,\n",
       " 1,\n",
       " 7,\n",
       " 8,\n",
       " 1,\n",
       " 7,\n",
       " 6,\n",
       " 5,\n",
       " 1,\n",
       " 2,\n",
       " 8,\n",
       " 9,\n",
       " 9,\n",
       " 8,\n",
       " 6,\n",
       " 8,\n",
       " 0,\n",
       " 10,\n",
       " 11,\n",
       " 3,\n",
       " 5,\n",
       " 4,\n",
       " 0,\n",
       " 0,\n",
       " 10,\n",
       " 7,\n",
       " 7,\n",
       " 4,\n",
       " 0,\n",
       " 9,\n",
       " 0,\n",
       " 8,\n",
       " 3,\n",
       " 2,\n",
       " 8,\n",
       " 0,\n",
       " 8,\n",
       " 8,\n",
       " 0,\n",
       " 6,\n",
       " 6,\n",
       " 8,\n",
       " 6,\n",
       " 2,\n",
       " 8,\n",
       " 9,\n",
       " 6,\n",
       " 7,\n",
       " 7,\n",
       " 10,\n",
       " 1,\n",
       " 5,\n",
       " 0,\n",
       " 3,\n",
       " 2,\n",
       " 2,\n",
       " 11,\n",
       " 7,\n",
       " 2,\n",
       " 10,\n",
       " 9,\n",
       " 9,\n",
       " 10,\n",
       " 0,\n",
       " 5,\n",
       " 5,\n",
       " 5,\n",
       " 7,\n",
       " 0,\n",
       " 6,\n",
       " 3,\n",
       " 9,\n",
       " 5,\n",
       " 10,\n",
       " 10,\n",
       " 3,\n",
       " 6,\n",
       " 7,\n",
       " 2,\n",
       " 2,\n",
       " 10,\n",
       " 11,\n",
       " 5,\n",
       " 7,\n",
       " 2,\n",
       " 11,\n",
       " 6,\n",
       " 8,\n",
       " 4,\n",
       " 6,\n",
       " 11,\n",
       " 7,\n",
       " 2,\n",
       " 6,\n",
       " 0,\n",
       " 4,\n",
       " 9,\n",
       " 3,\n",
       " 9,\n",
       " 2,\n",
       " 8,\n",
       " 4,\n",
       " 5,\n",
       " 10,\n",
       " 9,\n",
       " 5,\n",
       " 9,\n",
       " 4,\n",
       " 4,\n",
       " 9,\n",
       " 8,\n",
       " 2,\n",
       " 0,\n",
       " 3,\n",
       " 4,\n",
       " 2,\n",
       " 4,\n",
       " 6,\n",
       " 10,\n",
       " 8,\n",
       " 8,\n",
       " 1,\n",
       " 4,\n",
       " 1,\n",
       " 7,\n",
       " 0,\n",
       " 5,\n",
       " 6,\n",
       " 11,\n",
       " 11,\n",
       " 11,\n",
       " 7,\n",
       " 4,\n",
       " 3,\n",
       " 3,\n",
       " 5,\n",
       " 8,\n",
       " 11,\n",
       " 5,\n",
       " 2,\n",
       " 3,\n",
       " 2,\n",
       " 5,\n",
       " 4,\n",
       " 4,\n",
       " 7,\n",
       " 10,\n",
       " 1,\n",
       " 9,\n",
       " 6,\n",
       " 3,\n",
       " 8,\n",
       " 11,\n",
       " 7,\n",
       " 3,\n",
       " 3,\n",
       " 2,\n",
       " 10,\n",
       " 7,\n",
       " 11,\n",
       " 8,\n",
       " 11,\n",
       " 7,\n",
       " 9,\n",
       " 3,\n",
       " 11,\n",
       " 8,\n",
       " 0,\n",
       " 4,\n",
       " 3,\n",
       " 10,\n",
       " 7,\n",
       " 6,\n",
       " 6,\n",
       " 3,\n",
       " 0,\n",
       " 5,\n",
       " 2,\n",
       " 7,\n",
       " 9,\n",
       " 10,\n",
       " 0,\n",
       " 2,\n",
       " 11,\n",
       " 1,\n",
       " 3,\n",
       " 7,\n",
       " 9,\n",
       " 8,\n",
       " 4,\n",
       " 0,\n",
       " 5,\n",
       " 6,\n",
       " 4,\n",
       " 9,\n",
       " 1,\n",
       " 9,\n",
       " 0,\n",
       " 10,\n",
       " 6,\n",
       " 6,\n",
       " 1,\n",
       " 6,\n",
       " 0,\n",
       " 9,\n",
       " 3,\n",
       " 4,\n",
       " 7,\n",
       " 4,\n",
       " 7,\n",
       " 10,\n",
       " 2,\n",
       " 1,\n",
       " 9,\n",
       " 5,\n",
       " 6,\n",
       " 4,\n",
       " 11,\n",
       " 10,\n",
       " 2,\n",
       " 8,\n",
       " 0,\n",
       " 10,\n",
       " 8,\n",
       " 7,\n",
       " 7,\n",
       " 0,\n",
       " 6,\n",
       " 6,\n",
       " 9,\n",
       " 5,\n",
       " 5,\n",
       " 4,\n",
       " 1,\n",
       " 0,\n",
       " 8,\n",
       " 8,\n",
       " 2,\n",
       " 0,\n",
       " 7,\n",
       " 3,\n",
       " 6,\n",
       " 8,\n",
       " 2,\n",
       " 9,\n",
       " 2,\n",
       " 2,\n",
       " 5,\n",
       " 11,\n",
       " 11,\n",
       " 8,\n",
       " 8,\n",
       " 6,\n",
       " 8,\n",
       " 8,\n",
       " 2,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 10,\n",
       " 9,\n",
       " 5,\n",
       " 1,\n",
       " 2,\n",
       " 6,\n",
       " 3,\n",
       " 11,\n",
       " 4,\n",
       " 1,\n",
       " 1,\n",
       " 9,\n",
       " 5,\n",
       " 11,\n",
       " 0,\n",
       " 10,\n",
       " 8,\n",
       " 8,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 11,\n",
       " 6,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 7,\n",
       " 5,\n",
       " 7,\n",
       " 7,\n",
       " 1,\n",
       " 8,\n",
       " 2,\n",
       " 7,\n",
       " 5,\n",
       " 9,\n",
       " 8,\n",
       " 8,\n",
       " 6,\n",
       " 10,\n",
       " 4,\n",
       " 3,\n",
       " 7,\n",
       " 7,\n",
       " 11,\n",
       " 9,\n",
       " 4,\n",
       " 9,\n",
       " 5,\n",
       " 8,\n",
       " 2,\n",
       " 10,\n",
       " 3,\n",
       " 10,\n",
       " 6,\n",
       " 7,\n",
       " 8,\n",
       " 7,\n",
       " 1,\n",
       " 10,\n",
       " 2,\n",
       " 5,\n",
       " 10,\n",
       " 0,\n",
       " 1,\n",
       " 7,\n",
       " 1,\n",
       " 11,\n",
       " 6,\n",
       " 2,\n",
       " 7,\n",
       " 8,\n",
       " 7,\n",
       " 0,\n",
       " 11,\n",
       " 2,\n",
       " 1,\n",
       " 3,\n",
       " 7,\n",
       " 6,\n",
       " 4,\n",
       " 3,\n",
       " 4,\n",
       " 7,\n",
       " 11,\n",
       " 1,\n",
       " 11,\n",
       " 4,\n",
       " 1,\n",
       " 4,\n",
       " 11,\n",
       " 3,\n",
       " 10,\n",
       " 1,\n",
       " 9,\n",
       " 5,\n",
       " 0,\n",
       " 6,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 5,\n",
       " 9,\n",
       " 5,\n",
       " 2,\n",
       " 0,\n",
       " 8,\n",
       " 7,\n",
       " 9,\n",
       " 3,\n",
       " 5,\n",
       " 10,\n",
       " 0,\n",
       " 11,\n",
       " 3,\n",
       " 5,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 10,\n",
       " 11,\n",
       " 6,\n",
       " 4,\n",
       " 3,\n",
       " 10,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 5,\n",
       " 4,\n",
       " 9,\n",
       " 1,\n",
       " 1,\n",
       " 6,\n",
       " 2,\n",
       " 8,\n",
       " 7,\n",
       " 6,\n",
       " 6,\n",
       " 9,\n",
       " 10,\n",
       " 8,\n",
       " 0,\n",
       " 11,\n",
       " 11,\n",
       " 11,\n",
       " 11,\n",
       " 8,\n",
       " 4,\n",
       " ...]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(X_test_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_test  {'month': array([10,  8,  0, ...,  8, 11,  4], dtype=int64), 'hour': array([10, 15, 10, ..., 20, 15,  0], dtype=int64), 'daym': array([11, 14, 16, ..., 15, 22, 25], dtype=int64), 'Route': array([ 9,  6,  5, ...,  3,  8, 11]), 'day': array([3, 0, 5, ..., 2, 0, 1], dtype=int64), 'Direction': array([0, 0, 0, ..., 3, 3, 2]), 'year': array([1, 0, 1, ..., 1, 5, 1], dtype=int64)}\n"
     ]
    }
   ],
   "source": [
    "print(\"X_test \", X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predictions on training set\n",
    "\n",
    "preds = saved_model.predict(X_test, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(590520, 1)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\personal\\manning\\tf2_test\\notebooks\\try_tf2\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAASiUlEQVR4nO3df6zd9X3f8eerOMm8tCEGwhXCdGbD20qSpS2eQes23cUddpNpZBKRvLHgVp6sZXTKJKQV+sesBSHBH1k6spLKSixMhkosmtZeO4Ys2F02ld9bEgcYwysMXKygxB7FTGExfe+P87nj4Fx/7vH9cS6X83xIR/ec9/f7+X4/72vrvu73xzk3VYUkSWfyEys9AUnSO5tBIUnqMigkSV0GhSSpy6CQJHWtWekJLLULLrigNmzYsODxr7/+Ou9///uXbkKrwKT1PGn9gj1PisX0/OSTT36/qj4017J3XVBs2LCBJ554YsHjZ2ZmmJ6eXroJrQKT1vOk9Qv2PCkW03OS/3WmZZ56kiR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdb3r3pm9WIf/+FV++aY/GPt+X7jtk2PfpySNwiMKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1DVyUCQ5J8l/S/L77fV5SQ4lea59XTe07s1JjiR5NsnWofoVSQ63ZXckSau/L8nXW/3RJBuGxuxo+3guyY6laFqSNLqzOaL4HPDM0OubgAeraiPwYHtNksuB7cCHgW3AnUnOaWO+DOwCNrbHtlbfCZyoqsuALwK3t22dB+wGrgQ2A7uHA0mStPxGCook64FPAl8ZKl8D7GvP9wGfGqrfW1VvVNXzwBFgc5KLgA9U1cNVVcDdp42Z3dZ9wJZ2tLEVOFRVx6vqBHCIt8JFkjQGo/4p1N8A/jnwU0O1qao6BlBVx5Jc2OoXA48MrXe01X7Unp9enx3zUtvWqSSvAucP1+cY8/8l2cXgSIWpqSlmZmZGbOvHTa2FGz96asHjF2oxc16skydPruj+x23S+gV7nhTL1fO8QZHk7wCvVNWTSaZH2GbmqFWnvtAxbxWq9gB7ADZt2lTT06NMc25fuucAXzg8/j8l/sJ102Pf56yZmRkW8z1bbSatX7DnSbFcPY9y6ukXgL+b5AXgXuDjSf4t8L12Oon29ZW2/lHgkqHx64GXW339HPW3jUmyBjgXON7ZliRpTOYNiqq6uarWV9UGBhepH6qqfwgcBGbvQtoBHGjPDwLb251MlzK4aP1YO031WpKr2vWH608bM7uta9s+CngAuDrJunYR++pWkySNyWLOsdwG7E+yE3gR+DRAVT2VZD/wNHAKuKGq3mxjPgvcBawF7m8PgK8CX0tyhMGRxPa2reNJbgEeb+t9vqqOL2LOkqSzdFZBUVUzwEx7/gNgyxnWuxW4dY76E8BH5qj/kBY0cyzbC+w9m3lKkpaO78yWJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeqaNyiS/JkkjyX5dpKnkvzLVj8vyaEkz7Wv64bG3JzkSJJnk2wdql+R5HBbdkeStPr7kny91R9NsmFozI62j+eS7FjK5iVJ8xvliOIN4ONV9THgZ4FtSa4CbgIerKqNwIPtNUkuB7YDHwa2AXcmOadt68vALmBje2xr9Z3Aiaq6DPgicHvb1nnAbuBKYDOweziQJEnLb96gqIGT7eV72qOAa4B9rb4P+FR7fg1wb1W9UVXPA0eAzUkuAj5QVQ9XVQF3nzZmdlv3AVva0cZW4FBVHa+qE8Ah3goXSdIYrBllpXZE8CRwGfCbVfVokqmqOgZQVceSXNhWvxh4ZGj40Vb7UXt+en12zEttW6eSvAqcP1yfY8zw/HYxOFJhamqKmZmZUdqa09RauPGjpxY8fqEWM+fFOnny5Iruf9wmrV+w50mxXD2PFBRV9Sbws0k+CPxuko90Vs9cm+jUFzpmeH57gD0AmzZtqunp6c70+r50zwG+cHikb8uSeuG66bHvc9bMzAyL+Z6tNpPWL9jzpFiuns/qrqeq+t/ADIPTP99rp5NoX19pqx0FLhkath54udXXz1F/25gka4BzgeOdbUmSxmSUu54+1I4kSLIW+EXgvwMHgdm7kHYAB9rzg8D2difTpQwuWj/WTlO9luSqdv3h+tPGzG7rWuChdh3jAeDqJOvaReyrW02SNCajnGO5CNjXrlP8BLC/qn4/ycPA/iQ7gReBTwNU1VNJ9gNPA6eAG9qpK4DPAncBa4H72wPgq8DXkhxhcCSxvW3reJJbgMfbep+vquOLaViSdHbmDYqq+g7wc3PUfwBsOcOYW4Fb56g/AfzY9Y2q+iEtaOZYthfYO988JUnLw3dmS5K6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1zRsUSS5J8h+TPJPkqSSfa/XzkhxK8lz7um5ozM1JjiR5NsnWofoVSQ63ZXckSau/L8nXW/3RJBuGxuxo+3guyY6lbF6SNL9RjihOATdW1c8AVwE3JLkcuAl4sKo2Ag+217Rl24EPA9uAO5Oc07b1ZWAXsLE9trX6TuBEVV0GfBG4vW3rPGA3cCWwGdg9HEiSpOU3b1BU1bGq+q/t+WvAM8DFwDXAvrbaPuBT7fk1wL1V9UZVPQ8cATYnuQj4QFU9XFUF3H3amNlt3QdsaUcbW4FDVXW8qk4Ah3grXCRJY7DmbFZup4R+DngUmKqqYzAIkyQXttUuBh4ZGna01X7Unp9enx3zUtvWqSSvAucP1+cYMzyvXQyOVJiammJmZuZs2nqbqbVw40dPLXj8Qi1mzot18uTJFd3/uE1av2DPk2K5eh45KJL8JPA7wD+rqj9plxfmXHWOWnXqCx3zVqFqD7AHYNOmTTU9PX2muc3rS/cc4AuHzyo/l8QL102PfZ+zZmZmWMz3bLWZtH7BnifFcvU80l1PSd7DICTuqapvtPL32ukk2tdXWv0ocMnQ8PXAy62+fo7628YkWQOcCxzvbEuSNCaj3PUU4KvAM1X1r4YWHQRm70LaARwYqm9vdzJdyuCi9WPtNNVrSa5q27z+tDGz27oWeKhdx3gAuDrJunYR++pWkySNySjnWH4B+AxwOMm3Wu3XgduA/Ul2Ai8CnwaoqqeS7AeeZnDH1A1V9WYb91ngLmAtcH97wCCIvpbkCIMjie1tW8eT3AI83tb7fFUdX2CvkqQFmDcoquq/MPe1AoAtZxhzK3DrHPUngI/MUf8hLWjmWLYX2DvfPCVJy8N3ZkuSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdc0bFEn2JnklyXeHauclOZTkufZ13dCym5McSfJskq1D9SuSHG7L7kiSVn9fkq+3+qNJNgyN2dH28VySHUvVtCRpdKMcUdwFbDutdhPwYFVtBB5sr0lyObAd+HAbc2eSc9qYLwO7gI3tMbvNncCJqroM+CJwe9vWecBu4EpgM7B7OJAkSeMxb1BU1TeB46eVrwH2tef7gE8N1e+tqjeq6nngCLA5yUXAB6rq4aoq4O7Txsxu6z5gSzva2AocqqrjVXUCOMSPB5YkaZmtWeC4qao6BlBVx5Jc2OoXA48MrXe01X7Unp9enx3zUtvWqSSvAucP1+cY8zZJdjE4WmFqaoqZmZkFtgVTa+HGj55a8PiFWsycF+vkyZMruv9xm7R+wZ4nxXL1vNCgOJPMUatOfaFj3l6s2gPsAdi0aVNNT0/PO9Ez+dI9B/jC4aX+tszvheumx77PWTMzMyzme7baTFq/YM+TYrl6XuhdT99rp5NoX19p9aPAJUPrrQdebvX1c9TfNibJGuBcBqe6zrQtSdIYLTQoDgKzdyHtAA4M1be3O5kuZXDR+rF2muq1JFe16w/XnzZmdlvXAg+16xgPAFcnWdcuYl/dapKkMZr3HEuS3wamgQuSHGVwJ9JtwP4kO4EXgU8DVNVTSfYDTwOngBuq6s22qc8yuINqLXB/ewB8FfhakiMMjiS2t20dT3IL8Hhb7/NVdfpFdUnSMps3KKrq759h0ZYzrH8rcOsc9SeAj8xR/yEtaOZYthfYO98cJUnLx3dmS5K6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklS17x/ClWSdHY23PQHK7Lfu7a9f1m26xGFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktTlhwK+Q6zUh4jB8n2QmKR3B4NCHP7jV/nlFQiqF2775Nj3KensGRRaMe+2j2KW3q0MCk2clTqCAo+itDp5MVuS1OURhTRGnm7TamRQSBPA021ajFURFEm2Af8aOAf4SlXdtsJTkjQij6JWv3d8UCQ5B/hN4G8DR4HHkxysqqdXdmaS3slW8ijq3WY1XMzeDBypqj+qqv8L3Atcs8JzkqSJkapa6Tl0JbkW2FZV/6i9/gxwZVX96tA6u4Bd7eVfAp5dxC4vAL6/iPGr0aT1PGn9gj1PisX0/Oeq6kNzLXjHn3oCMkftbelWVXuAPUuys+SJqtq0FNtaLSat50nrF+x5UixXz6vh1NNR4JKh1+uBl1doLpI0cVZDUDwObExyaZL3AtuBgys8J0maGO/4U09VdSrJrwIPMLg9dm9VPbWMu1ySU1irzKT1PGn9gj1PimXp+R1/MVuStLJWw6knSdIKMigkSV0TGRRJtiV5NsmRJDfNsTxJ7mjLv5Pk51dinktphJ6va71+J8kfJvnYSsxzKc3X89B6fzXJm+09O6vaKD0nmU7yrSRPJflP457jUhvh//a5Sf5dkm+3nn9lJea5VJLsTfJKku+eYfnS//yqqol6MLgg/j+BPw+8F/g2cPlp63wCuJ/BeziuAh5d6XmPoee/Bqxrz39pEnoeWu8h4N8D1670vMfw7/xB4Gngp9vrC1d63mPo+deB29vzDwHHgfeu9NwX0fPfBH4e+O4Zli/5z69JPKIY5SNBrgHuroFHgA8muWjcE11C8/ZcVX9YVSfay0cYvF9lNRv1o1/+KfA7wCvjnNwyGaXnfwB8o6peBKiq1d73KD0X8FNJAvwkg6A4Nd5pLp2q+iaDHs5kyX9+TWJQXAy8NPT6aKud7Tqrydn2s5PBbySr2bw9J7kY+HvAb41xXstplH/nvwisSzKT5Mkk149tdstjlJ7/DfAzDN6oexj4XFX96XimtyKW/OfXO/59FMtg3o8EGXGd1WTkfpL8LQZB8deXdUbLb5SefwP4tap6c/DL5qo3Ss9rgCuALcBa4OEkj1TV/1juyS2TUXreCnwL+DjwF4BDSf5zVf3Jck9uhSz5z69JDIpRPhLk3faxISP1k+SvAF8BfqmqfjCmuS2XUXreBNzbQuIC4BNJTlXV741niktu1P/b36+q14HXk3wT+BiwWoNilJ5/BbitBifwjyR5HvjLwGPjmeLYLfnPr0k89TTKR4IcBK5vdw9cBbxaVcfGPdElNG/PSX4a+AbwmVX82+WweXuuqkurakNVbQDuA/7JKg4JGO3/9gHgbyRZk+TPAlcCz4x5nktplJ5fZHAERZIpBp8w/UdjneV4LfnPr4k7oqgzfCRIkn/clv8WgztgPgEcAf4Pg99IVq0Re/4XwPnAne037FO1ij95c8Se31VG6bmqnknyH4DvAH/K4C9Gznmb5Wow4r/zLcBdSQ4zOC3za1W1aj9+PMlvA9PABUmOAruB98Dy/fzyIzwkSV2TeOpJknQWDApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkrv8HP/0fDcJ0ZYIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test[\"predict\"] = preds\n",
    "test.predict[:5]\n",
    "if verboseout:\n",
    "    test.predict.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAUqElEQVR4nO3df6zd9X3f8eerOMm8NCEGwhXCdGaLt5XES1o8g5Ztuo0n20mnkUlEcseCW3myltEpkyyt0D+GFoQU/shSkY10VrAwGSuxaDKzdpRZsLtsKj+zJXGAMrzAwMOKldijOFNYLn3vj/NxOTjXn3vs++Nwc54P6eic8/5+P9/v542v78vfH+eQqkKSpDP5mXFPQJL01mZQSJK6DApJUpdBIUnqMigkSV2rxj2BxXbRRRfVunXrznn8D3/4Q975zncu3oRWgEnredL6BXueFAvp+Rvf+Mb3q+q9cy37qQuKdevW8eSTT57z+JmZGaanpxdvQivApPU8af2CPU+KhfSc5H+daZmnniRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0/dZ/MXqhD//sVfvXG3x/3NJbV7g2zE9XzOPt94bO/PJb9SgvhEYUkqWukoEjyQpJDSb6Z5MlWuyDJwSTPtec1Q+vflORwkmeTbB2qX9m2czjJ7UnS6u9I8pVWfyzJuqExO9o+nkuyY7EalySN5myOKH6pqj5UVRvb+xuBh6pqPfBQe0+SK4DtwPuBbcAdSc5rY74I7ALWt8e2Vt8JnKiq9wGfB25r27oAuBm4CtgE3DwcSJKkpbeQU0/XAPva633Ax4fq91bVa1X1PHAY2JTkEuDdVfVIVRVw92ljTm3rPmBzO9rYChysquNVdQI4yBvhIklaBqNezC7gPyYp4F9X1R5gqqqOAlTV0SQXt3UvBR4dGnuk1X7cXp9ePzXmpbat2SSvABcO1+cY86eS7GJwpMLU1BQzMzMjtvWTplYPLnZOkknreZz9LuRncyFOnjw5tn2Piz0vnlGD4sNV9XILg4NJ/qizbuaoVad+rmPeKAyCaw/Axo0bayHfQf+Few7wuUOTdTPY7g2zE9XzOPt94brpsezX/zfDZFiqnkc69VRVL7fnY8DXGFwv+F47nUR7PtZWPwJcNjR8LfByq6+do/6mMUlWAecDxzvbkiQtk3mDIsk7k7zr1GtgC/Ad4H7g1F1IO4AD7fX9wPZ2J9PlDC5aP95OU72a5Op2/eH608ac2ta1wMPtOsaDwJYka9pF7C2tJklaJqMcf08BX2t3sq4C/m1V/UGSJ4D9SXYCLwKfAKiqp5LsB54GZoEbqur1tq1PAXcBq4EH2gPgTuDLSQ4zOJLY3rZ1PMktwBNtvc9U1fEF9CtJOkvzBkVVfRf44Bz1HwCbzzDmVuDWOepPAh+Yo/4jWtDMsWwvsHe+eUqSloafzJYkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0jB0WS85L89yS/195fkORgkufa85qhdW9KcjjJs0m2DtWvTHKoLbs9SVr9HUm+0uqPJVk3NGZH28dzSXYsRtOSpNGdzRHFp4Fnht7fCDxUVeuBh9p7klwBbAfeD2wD7khyXhvzRWAXsL49trX6TuBEVb0P+DxwW9vWBcDNwFXAJuDm4UCSJC29kYIiyVrgl4EvDZWvAfa11/uAjw/V762q16rqeeAwsCnJJcC7q+qRqirg7tPGnNrWfcDmdrSxFThYVcer6gRwkDfCRZK0DFaNuN5vAf8UeNdQbaqqjgJU1dEkF7f6pcCjQ+sdabUft9en10+NealtazbJK8CFw/U5xvypJLsYHKkwNTXFzMzMiG39pKnVsHvD7DmPX4kmredx9ruQn82FOHny5Nj2PS72vHjmDYokfxs4VlXfSDI9wjYzR6069XMd80ahag+wB2Djxo01PT3KNOf2hXsO8LlDo+bnT4fdG2Ynqudx9vvCddNj2e/MzAwL+XuxEtnz4hnl1NOHgb+T5AXgXuAjSf4N8L12Oon2fKytfwS4bGj8WuDlVl87R/1NY5KsAs4Hjne2JUlaJvMGRVXdVFVrq2odg4vUD1fV3wfuB07dhbQDONBe3w9sb3cyXc7govXj7TTVq0mubtcfrj9tzKltXdv2UcCDwJYka9pF7C2tJklaJgs5/v4ssD/JTuBF4BMAVfVUkv3A08AscENVvd7GfAq4C1gNPNAeAHcCX05ymMGRxPa2reNJbgGeaOt9pqqOL2DOkqSzdFZBUVUzwEx7/QNg8xnWuxW4dY76k8AH5qj/iBY0cyzbC+w9m3lKkhaPn8yWJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEld8wZFkj+T5PEk30ryVJJ/3uoXJDmY5Ln2vGZozE1JDid5NsnWofqVSQ61ZbcnSau/I8lXWv2xJOuGxuxo+3guyY7FbF6SNL9RjiheAz5SVR8EPgRsS3I1cCPwUFWtBx5q70lyBbAdeD+wDbgjyXltW18EdgHr22Nbq+8ETlTV+4DPA7e1bV0A3AxcBWwCbh4OJEnS0ps3KGrgZHv7tvYo4BpgX6vvAz7eXl8D3FtVr1XV88BhYFOSS4B3V9UjVVXA3aeNObWt+4DN7WhjK3Cwqo5X1QngIG+EiyRpGYx0jSLJeUm+CRxj8Iv7MWCqqo4CtOeL2+qXAi8NDT/Sape216fX3zSmqmaBV4ALO9uSJC2TVaOsVFWvAx9K8h7ga0k+0Fk9c22iUz/XMW/sMNnF4JQWU1NTzMzMdKbXN7Uadm+YPefxK9Gk9TzOfhfys7kQJ0+eHNu+x8WeF89IQXFKVf2fJDMMTv98L8klVXW0nVY61lY7Alw2NGwt8HKrr52jPjzmSJJVwPnA8VafPm3MzBzz2gPsAdi4cWNNT0+fvsrIvnDPAT536Kz+s6x4uzfMTlTP4+z3heumx7LfmZkZFvL3YiWy58Uzyl1P721HEiRZDfwt4I+A+4FTdyHtAA601/cD29udTJczuGj9eDs99WqSq9v1h+tPG3NqW9cCD7frGA8CW5KsaRext7SaJGmZjPLPqkuAfe3OpZ8B9lfV7yV5BNifZCfwIvAJgKp6Ksl+4GlgFrihnboC+BRwF7AaeKA9AO4EvpzkMIMjie1tW8eT3AI80db7TFUdX0jDkqSzM29QVNW3gV+Yo/4DYPMZxtwK3DpH/UngJ65vVNWPaEEzx7K9wN755ilJWhp+MluS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkrnmDIsllSf5TkmeSPJXk061+QZKDSZ5rz2uGxtyU5HCSZ5NsHapfmeRQW3Z7krT6O5J8pdUfS7JuaMyOto/nkuxYzOYlSfMb5YhiFthdVT8PXA3ckOQK4EbgoapaDzzU3tOWbQfeD2wD7khyXtvWF4FdwPr22NbqO4ETVfU+4PPAbW1bFwA3A1cBm4CbhwNJkrT05g2KqjpaVf+tvX4VeAa4FLgG2NdW2wd8vL2+Bri3ql6rqueBw8CmJJcA766qR6qqgLtPG3NqW/cBm9vRxlbgYFUdr6oTwEHeCBdJ0jJYdTYrt1NCvwA8BkxV1VEYhEmSi9tqlwKPDg070mo/bq9Pr58a81Lb1mySV4ALh+tzjBme1y4GRypMTU0xMzNzNm29ydRq2L1h9pzHr0ST1vM4+13Iz+ZCnDx5cmz7Hhd7XjwjB0WSnwV+F/gnVfXH7fLCnKvOUatO/VzHvFGo2gPsAdi4cWNNT0+faW7z+sI9B/jcobPKzxVv94bZiep5nP2+cN30WPY7MzPDQv5erET2vHhGuuspydsYhMQ9VfXVVv5eO51Eez7W6keAy4aGrwVebvW1c9TfNCbJKuB84HhnW5KkZTLKXU8B7gSeqap/MbTofuDUXUg7gAND9e3tTqbLGVy0frydpno1ydVtm9efNubUtq4FHm7XMR4EtiRZ0y5ib2k1SdIyGeX4+8PAJ4FDSb7Zar8JfBbYn2Qn8CLwCYCqeirJfuBpBndM3VBVr7dxnwLuAlYDD7QHDILoy0kOMziS2N62dTzJLcATbb3PVNXxc+xVknQO5g2KqvqvzH2tAGDzGcbcCtw6R/1J4ANz1H9EC5o5lu0F9s43T0nS0vCT2ZKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqWveoEiyN8mxJN8Zql2Q5GCS59rzmqFlNyU5nOTZJFuH6lcmOdSW3Z4krf6OJF9p9ceSrBsas6Pt47kkOxaraUnS6EY5orgL2HZa7UbgoapaDzzU3pPkCmA78P425o4k57UxXwR2Aevb49Q2dwInqup9wOeB29q2LgBuBq4CNgE3DweSJGl5zBsUVfV14Php5WuAfe31PuDjQ/V7q+q1qnoeOAxsSnIJ8O6qeqSqCrj7tDGntnUfsLkdbWwFDlbV8ao6ARzkJwNLkrTEVp3juKmqOgpQVUeTXNzqlwKPDq13pNV+3F6fXj815qW2rdkkrwAXDtfnGPMmSXYxOFphamqKmZmZc2wLplbD7g2z5zx+JZq0nsfZ70J+Nhfi5MmTY9v3uNjz4jnXoDiTzFGrTv1cx7y5WLUH2AOwcePGmp6enneiZ/KFew7wuUOL/Z/lrW33htmJ6nmc/b5w3fRY9jszM8NC/l6sRPa8eM71rqfvtdNJtOdjrX4EuGxovbXAy62+do76m8YkWQWcz+BU15m2JUlaRucaFPcDp+5C2gEcGKpvb3cyXc7govXj7TTVq0mubtcfrj9tzKltXQs83K5jPAhsSbKmXcTe0mqSpGU07/F3kt8BpoGLkhxhcCfSZ4H9SXYCLwKfAKiqp5LsB54GZoEbqur1tqlPMbiDajXwQHsA3Al8OclhBkcS29u2jie5BXiirfeZqjr9orokaYnNGxRV9StnWLT5DOvfCtw6R/1J4ANz1H9EC5o5lu0F9s43R0nS0vGT2ZKkLoNCktRlUEiSuibn5nnpLWDdjb8/lv3u3jDLr45p3+MyiT3fte2dS7JdjygkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkrhURFEm2JXk2yeEkN457PpI0Sd7yQZHkPOBfAR8FrgB+JckV452VJE2Ot3xQAJuAw1X13ar6f8C9wDVjnpMkTYxU1bjn0JXkWmBbVf2D9v6TwFVV9etD6+wCdrW3fwl4dgG7vAj4/gLGr0ST1vOk9Qv2PCkW0vOfq6r3zrVg1bnPZ9lkjtqb0q2q9gB7FmVnyZNVtXExtrVSTFrPk9Yv2POkWKqeV8KppyPAZUPv1wIvj2kukjRxVkJQPAGsT3J5krcD24H7xzwnSZoYb/lTT1U1m+TXgQeB84C9VfXUEu5yUU5hrTCT1vOk9Qv2PCmWpOe3/MVsSdJ4rYRTT5KkMTIoJEldExkU830lSAZub8u/neQXxzHPxTRCz9e1Xr+d5A+TfHAc81xMo371S5K/muT19pmdFW2UnpNMJ/lmkqeS/OflnuNiG+Fn+/wk/z7Jt1rPvzaOeS6WJHuTHEvynTMsX/zfX1U1UQ8GF8T/J/DngbcD3wKuOG2djwEPMPgMx9XAY+Oe9zL0/NeANe31Ryeh56H1Hgb+A3DtuOe9DH/O7wGeBn6uvb943PNehp5/E7itvX4vcBx4+7jnvoCe/ybwi8B3zrB80X9/TeIRxShfCXINcHcNPAq8J8klyz3RRTRvz1X1h1V1or19lMHnVVayUb/65R8DvwscW87JLZFRev57wFer6kWAqlrpfY/ScwHvShLgZxkExezyTnPxVNXXGfRwJov++2sSg+JS4KWh90da7WzXWUnOtp+dDP5FspLN23OSS4G/C/z2Ms5rKY3y5/wXgTVJZpJ8I8n1yza7pTFKz/8S+HkGH9Q9BHy6qv5keaY3Fov+++st/zmKJTDvV4KMuM5KMnI/SX6JQVD89SWd0dIbpeffAn6jql4f/GNzxRul51XAlcBmYDXwSJJHq+p/LPXklsgoPW8Fvgl8BPgLwMEk/6Wq/nipJzcmi/77axKDYpSvBPlp+9qQkfpJ8leALwEfraofLNPclsooPW8E7m0hcRHwsSSzVfXvlmeKi27Un+3vV9UPgR8m+TrwQWClBsUoPf8a8NkanMA/nOR54C8Djy/PFJfdov/+msRTT6N8Jcj9wPXt7oGrgVeq6uhyT3QRzdtzkp8Dvgp8cgX/63LYvD1X1eVVta6q1gH3Af9oBYcEjPazfQD4G0lWJfmzwFXAM8s8z8U0Ss8vMjiCIskUg2+Y/u6yznJ5Lfrvr4k7oqgzfCVIkn/Ylv82gztgPgYcBv4vg3+RrFgj9vzPgAuBO9q/sGdrBX/z5og9/1QZpeeqeibJHwDfBv4E+FJVzXmb5Uow4p/zLcBdSQ4xOC3zG1W1Yr9+PMnvANPARUmOADcDb4Ol+/3lV3hIkrom8dSTJOksGBSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXf8f0ZTxHpbu8uAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if verboseout:\n",
    "    test.predict.hist(bins=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\personal\\manning\\tf2_test\\notebooks\\try_tf2\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "917237    0\n",
       "346801    0\n",
       "513886    0\n",
       "289945    0\n",
       "358978    1\n",
       "Name: predround, dtype: int32"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get rounded predictions\n",
    "test[\"predround\"] = preds.round().astype(int)\n",
    "test.predround[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test target 0 578427\n",
      "test target 1 12093\n",
      "test predround 0 512275\n",
      "test predround 1 78245\n"
     ]
    }
   ],
   "source": [
    "print(\"test target 0\",test[test['target']==0].shape[0])\n",
    "print(\"test target 1\",test[test['target']==1].shape[0])\n",
    "print(\"test predround 0\",test[test['predround']==0].shape[0])\n",
    "print(\"test predround 1\",test[test['predround']==1].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12695\n",
      "percentage correct test\n",
      "0.87305\n"
     ]
    }
   ],
   "source": [
    "# get delta between predictions on training set and actual training target values\n",
    "# hand calculate accuracy on training set as ratio of (total training samples - wrong training predictions)/total training samples\n",
    "\n",
    "deltatr = abs(test.target[:100000] - test.predround[:100000])\n",
    "deltatr[:50]\n",
    "print(deltatr.sum())\n",
    "print(\"percentage correct test\")\n",
    "print((len(deltatr) - deltatr.sum())/len(deltatr))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "811053     0.000000\n",
       "1364770    0.542842\n",
       "1709203    0.000000\n",
       "923372     0.000000\n",
       "1290938    0.000000\n",
       "Name: predict, dtype: float32"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predict values for validation X values\n",
    "# X_valid, dvalid.target\n",
    "predval = model.predict(X_valid, batch_size=batch_size)\n",
    "dvalid[\"predround\"] = predval.round().astype(int)\n",
    "dvalid[\"predict\"] = predval\n",
    "#print(type(deltaval))\n",
    "#print(len(deltaval))\n",
    "dvalid.predict[:5]\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAZLklEQVR4nO3cf5DU933f8efLICtUtmTQjxsCpNAKt0GilqMrYuq2szEZwOofyDNocq5GkJgMjip1nJn7I5L/KI4YZqyZEHWkRqQ4MCANMWJku9BERL1I2Tqe8EPIg3VCMuFqUekMI8Y+gjl1RHXo3T/2c/WXy95nl9u9Xa329ZjZ2e++9/v57ucNd/u6749dRQRmZmaT+Vi7J2BmZh9uDgozM8tyUJiZWZaDwszMshwUZmaWNbPdE2i2m266KRYuXDjl8e+++y7XXXdd8ybUAbqt527rF9xzt2ik51deeeWnEXFztec+ckGxcOFCjh07NuXx5XKZUqnUvAl1gG7rudv6BffcLRrpWdL/nuw5H3oyM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLIcFGZmluWgMDOzrI/cJ7MbNfiTC/zWw3/R7mm0VP/Ssa7quZ39nv7Gv2vL65o1wnsUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLIcFGZmluWgMDOzrJpBIemXJB2V9ENJJyT9Qap/XdJPJB1Pt7sLYx6RNCTppKRVhfqdkgbTc09IUqpfK+nZVD8iaWFhzHpJp9JtfTObNzOz2ur5rqdLwOcjYlTSNcD3JR1Mzz0eEX9YXFnSEqAPuA34ZeCvJH06Ii4D24CNwGHgeWA1cBDYAJyPiFsl9QGPAb8paQ6wCegFAnhF0oGION9Y22ZmVq+aexRRMZoeXpNukRmyBtgbEZci4k1gCFgmaS5wfUQciogAngbuKYzZnZafA1akvY1VwEBEjKRwGKASLmZm1iJ1fXuspBnAK8CtwB9HxBFJXwAekrQOOAb0pzfzeVT2GMYNp9r7aXlinXT/NkBEjEm6ANxYrFcZU5zfRip7KvT09FAul+tpq6qeWZVvF+0m3dZzO/tt5GezEaOjo2177XZxz81TV1Ckw0Z3SPoU8F1Jt1M5jLSZyt7FZmAr8GVA1TaRqTPFMcX5bQe2A/T29kapVMq1k/Xknv1sHeyub1/vXzrWVT23s9/T95Xa8rrlcplGfi86kXtunqu66iki/h4oA6sj4p2IuBwRHwDfBJal1YaBBYVh84EzqT6/Sv2KMZJmAjcAI5ltmZlZi9Rz1dPNaU8CSbOA3wB+lM45jPsi8FpaPgD0pSuZFgGLgaMRcRa4KGl5Ov+wDthfGDN+RdNa4KV0HuMFYKWk2ZJmAytTzczMWqSe/e+5wO50nuJjwL6I+HNJz0i6g8qhoNPAVwAi4oSkfcDrwBjwYDp0BfAAsAuYReVqp/Grp3YAz0gaorIn0Ze2NSJpM/ByWu/RiBhpoF8zM7tKNYMiIl4FPlulfn9mzBZgS5X6MeD2KvX3gHsn2dZOYGeteZqZ2fTwJ7PNzCzLQWFmZlkOCjMzy3JQmJlZloPCzMyyHBRmZpbloDAzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWTWDQtIvSToq6YeSTkj6g1SfI2lA0ql0P7sw5hFJQ5JOSlpVqN8paTA994Qkpfq1kp5N9SOSFhbGrE+vcUrS+mY2b2ZmtdWzR3EJ+HxEfAa4A1gtaTnwMPBiRCwGXkyPkbQE6ANuA1YDT0makba1DdgILE631am+ATgfEbcCjwOPpW3NATYBdwHLgE3FQDIzs+lXMyiiYjQ9vCbdAlgD7E713cA9aXkNsDciLkXEm8AQsEzSXOD6iDgUEQE8PWHM+LaeA1akvY1VwEBEjETEeWCAX4SLmZm1QF3nKCTNkHQcOEfljfsI0BMRZwHS/S1p9XnA24Xhw6k2Ly1PrF8xJiLGgAvAjZltmZlZi8ysZ6WIuAzcIelTwHcl3Z5ZXdU2kalPdcwvXlDaSOWQFj09PZTL5cz08npmQf/SsSmP70Td1nM7+23kZ7MRo6OjbXvtdnHPzVNXUIyLiL+XVKZy+OcdSXMj4mw6rHQurTYMLCgMmw+cSfX5VerFMcOSZgI3ACOpXpowplxlXtuB7QC9vb1RKpUmrlK3J/fsZ+vgVf2zdLz+pWNd1XM7+z19X6ktr1sul2nk96ITuefmqeeqp5vTngSSZgG/AfwIOACMX4W0Htiflg8AfelKpkVUTlofTYenLkpans4/rJswZnxba4GX0nmMF4CVkmank9grU83MzFqknj+r5gK705VLHwP2RcSfSzoE7JO0AXgLuBcgIk5I2ge8DowBD6ZDVwAPALuAWcDBdAPYATwjaYjKnkRf2taIpM3Ay2m9RyNipJGGzczs6tQMioh4FfhslfrPgBWTjNkCbKlSPwb8g/MbEfEeKWiqPLcT2FlrnmZmNj38yWwzM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLIcFGZmluWgMDOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmJlZloPCzMyyagaFpAWS/lrSG5JOSPpqqn9d0k8kHU+3uwtjHpE0JOmkpFWF+p2SBtNzT0hSql8r6dlUPyJpYWHMekmn0m19M5s3M7PaZtaxzhjQHxE/kPRJ4BVJA+m5xyPiD4srS1oC9AG3Ab8M/JWkT0fEZWAbsBE4DDwPrAYOAhuA8xFxq6Q+4DHgNyXNATYBvUCk1z4QEecba9vMzOpVc48iIs5GxA/S8kXgDWBeZsgaYG9EXIqIN4EhYJmkucD1EXEoIgJ4GrinMGZ3Wn4OWJH2NlYBAxExksJhgEq4mJlZi9SzR/H/pUNCnwWOAJ8DHpK0DjhGZa/jPJUQOVwYNpxq76fliXXS/dsAETEm6QJwY7FeZUxxXhup7KnQ09NDuVy+mrau0DML+peOTXl8J+q2ntvZbyM/m40YHR1t22u3i3tunrqDQtIngG8DvxcRP5e0DdhM5ZDQZmAr8GVAVYZHps4Ux/yiELEd2A7Q29sbpVIp20vOk3v2s3XwqvKz4/UvHeuqntvZ7+n7Sm153XK5TCO/F53IPTdPXVc9SbqGSkjsiYjvAETEOxFxOSI+AL4JLEurDwMLCsPnA2dSfX6V+hVjJM0EbgBGMtsyM7MWqeeqJwE7gDci4o8K9bmF1b4IvJaWDwB96UqmRcBi4GhEnAUuSlqetrkO2F8YM35F01rgpXQe4wVgpaTZkmYDK1PNzMxapJ79788B9wODko6n2teAL0m6g8qhoNPAVwAi4oSkfcDrVK6YejBd8QTwALALmEXlaqeDqb4DeEbSEJU9ib60rRFJm4GX03qPRsTI1Fo1M7OpqBkUEfF9qp8reD4zZguwpUr9GHB7lfp7wL2TbGsnsLPWPM3MbHr4k9lmZpbloDAzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLIcFGZmluWgMDOzrJpBIWmBpL+W9IakE5K+mupzJA1IOpXuZxfGPCJpSNJJSasK9TslDabnnpCkVL9W0rOpfkTSwsKY9ek1Tkla38zmzcystnr2KMaA/oj4VWA58KCkJcDDwIsRsRh4MT0mPdcH3AasBp6SNCNtaxuwEVicbqtTfQNwPiJuBR4HHkvbmgNsAu4ClgGbioFkZmbTr2ZQRMTZiPhBWr4IvAHMA9YAu9Nqu4F70vIaYG9EXIqIN4EhYJmkucD1EXEoIgJ4esKY8W09B6xIexurgIGIGImI88AAvwgXMzNrgas6R5EOCX0WOAL0RMRZqIQJcEtabR7wdmHYcKrNS8sT61eMiYgx4AJwY2ZbZmbWIjPrXVHSJ4BvA78XET9PpxeqrlqlFpn6VMcU57aRyiEtenp6KJfLk82tpp5Z0L90bMrjO1G39dzOfhv52WzE6Oho2167Xdxz89QVFJKuoRISeyLiO6n8jqS5EXE2HVY6l+rDwILC8PnAmVSfX6VeHDMsaSZwAzCS6qUJY8oT5xcR24HtAL29vVEqlSauUrcn9+xn62Dd+fmR0L90rKt6bme/p+8rteV1y+UyjfxedCL33Dz1XPUkYAfwRkT8UeGpA8D4VUjrgf2Fel+6kmkRlZPWR9PhqYuSlqdtrpswZnxba4GX0nmMF4CVkmank9grU83MzFqknj+rPgfcDwxKOp5qXwO+AeyTtAF4C7gXICJOSNoHvE7liqkHI+JyGvcAsAuYBRxMN6gE0TOShqjsSfSlbY1I2gy8nNZ7NCJGptirmZlNQc2giIjvU/1cAcCKScZsAbZUqR8Dbq9Sf48UNFWe2wnsrDVPMzObHv5ktpmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLIcFGZmluWgMDOzLAeFmZllOSjMzCzLQWFmZlk1g0LSTknnJL1WqH1d0k8kHU+3uwvPPSJpSNJJSasK9TslDabnnpCkVL9W0rOpfkTSwsKY9ZJOpdv6ZjVtZmb1q2ePYhewukr98Yi4I92eB5C0BOgDbktjnpI0I62/DdgILE638W1uAM5HxK3A48BjaVtzgE3AXcAyYJOk2VfdoZmZNaRmUETE94CROre3BtgbEZci4k1gCFgmaS5wfUQciogAngbuKYzZnZafA1akvY1VwEBEjETEeWCA6oFlZmbTaGYDYx+StA44BvSnN/N5wOHCOsOp9n5anlgn3b8NEBFjki4ANxbrVcZcQdJGKnsr9PT0UC6Xp9xUzyzoXzo25fGdqNt6bme/jfxsNmJ0dLRtr90u7rl5phoU24DNQKT7rcCXAVVZNzJ1pjjmymLEdmA7QG9vb5RKpczU857cs5+tg43kZ+fpXzrWVT23s9/T95Xa8rrlcplGfi86kXtunild9RQR70TE5Yj4APgmlXMIUPmrf0Fh1fnAmVSfX6V+xRhJM4EbqBzqmmxbZmbWQlMKinTOYdwXgfErog4AfelKpkVUTlofjYizwEVJy9P5h3XA/sKY8Sua1gIvpfMYLwArJc1OJ7FXppqZmbVQzf1vSd8CSsBNkoapXIlUknQHlUNBp4GvAETECUn7gNeBMeDBiLicNvUAlSuoZgEH0w1gB/CMpCEqexJ9aVsjkjYDL6f1Ho2Iek+qm5lZk9QMioj4UpXyjsz6W4AtVerHgNur1N8D7p1kWzuBnbXmaGZm08efzDYzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLIcFGZmluWgMDOzLAeFmZllOSjMzCyrZlBI2inpnKTXCrU5kgYknUr3swvPPSJpSNJJSasK9TslDabnnpCkVL9W0rOpfkTSwsKY9ek1Tkla36ymzcysfvXsUewCVk+oPQy8GBGLgRfTYyQtAfqA29KYpyTNSGO2ARuBxek2vs0NwPmIuBV4HHgsbWsOsAm4C1gGbCoGkpmZtUbNoIiI7wEjE8prgN1peTdwT6G+NyIuRcSbwBCwTNJc4PqIOBQRATw9Ycz4tp4DVqS9jVXAQESMRMR5YIB/GFhmZjbNZk5xXE9EnAWIiLOSbkn1ecDhwnrDqfZ+Wp5YHx/zdtrWmKQLwI3FepUxV5C0kcreCj09PZTL5Sm2BT2zoH/p2JTHd6Ju67md/Tbys9mI0dHRtr12u7jn5plqUExGVWqRqU91zJXFiO3AdoDe3t4olUo1JzqZJ/fsZ+tgs/9ZPtz6l451Vc/t7Pf0faW2vG65XKaR34tO5J6bZ6q/Le9Impv2JuYC51J9GFhQWG8+cCbV51epF8cMS5oJ3EDlUNcwUJowpjzF+Zp9KCx8+C/a8rr9S8f4rTa9drt0Y8+7Vl83Ldud6uWxB4Dxq5DWA/sL9b50JdMiKietj6bDVBclLU/nH9ZNGDO+rbXAS+k8xgvASkmz00nslalmZmYtVHOPQtK3qPxlf5OkYSpXIn0D2CdpA/AWcC9ARJyQtA94HRgDHoyIy2lTD1C5gmoWcDDdAHYAz0gaorIn0Ze2NSJpM/ByWu/RiJh4Ut3MzKZZzaCIiC9N8tSKSdbfAmypUj8G3F6l/h4paKo8txPYWWuOZmY2ffzJbDMzy3JQmJlZloPCzMyyHBRmZpbloDAzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVlWQ0Eh6bSkQUnHJR1LtTmSBiSdSvezC+s/ImlI0klJqwr1O9N2hiQ9IUmpfq2kZ1P9iKSFjczXzMyuXjP2KH49Iu6IiN70+GHgxYhYDLyYHiNpCdAH3AasBp6SNCON2QZsBBan2+pU3wCcj4hbgceBx5owXzMzuwrTcehpDbA7Le8G7inU90bEpYh4ExgClkmaC1wfEYciIoCnJ4wZ39ZzwIrxvQ0zM2uNmQ2OD+B/SArgv0bEdqAnIs4CRMRZSbekdecBhwtjh1Pt/bQ8sT4+5u20rTFJF4AbgZ8WJyFpI5U9Enp6eiiXy1NuqGcW9C8dm/L4TtRtPXdbv+Ceu8Xo6GhD73+TaTQoPhcRZ1IYDEj6UWbdansCkannxlxZqATUdoDe3t4olUrZSec8uWc/Wwcb/WfpLP1Lx7qq527rF9xzt9i1+joaef+bTEOHniLiTLo/B3wXWAa8kw4nke7PpdWHgQWF4fOBM6k+v0r9ijGSZgI3ACONzNnMzK7OlINC0nWSPjm+DKwEXgMOAOvTauuB/Wn5ANCXrmRaROWk9dF0mOqipOXp/MO6CWPGt7UWeCmdxzAzsxZpZL+sB/huOrc8E/iziPhLSS8D+yRtAN4C7gWIiBOS9gGvA2PAgxFxOW3rAWAXMAs4mG4AO4BnJA1R2ZPoa2C+ZmY2BVMOioj4MfCZKvWfASsmGbMF2FKlfgy4vUr9PVLQmJlZe/iT2WZmluWgMDOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmJlZloPCzMyyHBRmZpbloDAzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWV1RFBIWi3ppKQhSQ+3ez5mZt3kQx8UkmYAfwx8AVgCfEnSkvbOysyse3zogwJYBgxFxI8j4v8Ce4E1bZ6TmVnXUES0ew5ZktYCqyPid9Lj+4G7IuKhwjobgY3p4T8DTjbwkjcBP21gfCfqtp67rV9wz92ikZ7/cUTcXO2JmVOfT8uoSu2KdIuI7cD2pryYdCwiepuxrU7RbT13W7/gnrvFdPXcCYeehoEFhcfzgTNtmouZWdfphKB4GVgsaZGkjwN9wIE2z8nMrGt86A89RcSYpIeAF4AZwM6IODGNL9mUQ1gdptt67rZ+wT13i2np+UN/MtvMzNqrEw49mZlZGzkozMwsqyuDotZXgqjiifT8q5J+rR3zbKY6er4v9fqqpL+V9Jl2zLOZ6v3qF0n/UtLl9JmdjlZPz5JKko5LOiHpf7Z6js1Wx8/2DZL+u6Qfpp5/ux3zbBZJOyWdk/TaJM83//0rIrrqRuWE+P8C/gnwceCHwJIJ69wNHKTyGY7lwJF2z7sFPf8rYHZa/kI39FxY7yXgeWBtu+fdgv/nTwGvA7+SHt/S7nm3oOevAY+l5ZuBEeDj7Z57Az3/W+DXgNcmeb7p71/duEdRz1eCrAGejorDwKckzW31RJuoZs8R8bcRcT49PEzl8yqdrN6vfvmPwLeBc62c3DSpp+d/D3wnIt4CiIhO77uengP4pCQBn6ASFGOtnWbzRMT3qPQwmaa/f3VjUMwD3i48Hk61q12nk1xtPxuo/EXSyWr2LGke8EXgT1o4r+lUz//zp4HZksqSXpG0rmWzmx719PxfgF+l8kHdQeCrEfFBa6bXFk1///rQf45iGtT8SpA61+kkdfcj6depBMW/ntYZTb96ev7PwO9HxOXKH5sdr56eZwJ3AiuAWcAhSYcj4u+me3LTpJ6eVwHHgc8D/xQYkPQ3EfHz6Z5cmzT9/asbg6KerwT5qH1tSF39SPoXwJ8CX4iIn7VobtOlnp57gb0pJG4C7pY0FhH/rTVTbLp6f7Z/GhHvAu9K+h7wGaBTg6Kenn8b+EZUDuAPSXoT+OfA0dZMseWa/v7VjYee6vlKkAPAunT1wHLgQkScbfVEm6hmz5J+BfgOcH8H/3VZVLPniFgUEQsjYiHwHPAfOjgkoL6f7f3Av5E0U9I/Au4C3mjxPJupnp7forIHhaQeKt8w/eOWzrK1mv7+1XV7FDHJV4JI+t30/J9QuQLmbmAI+D9U/iLpWHX2/J+AG4Gn0l/YY9HB37xZZ88fKfX0HBFvSPpL4FXgA+BPI6LqZZadoM7/583ALkmDVA7L/H5EdOzXj0v6FlACbpI0DGwCroHpe//yV3iYmVlWNx56MjOzq+CgMDOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmJlZ1v8Dagf191unvJkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if verboseout:\n",
    "    dvalid.predict.hist(bins=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "811053     0\n",
      "1364770    1\n",
      "1709203    0\n",
      "923372     0\n",
      "1290938    0\n",
      "2138815    0\n",
      "1896082    0\n",
      "1883851    0\n",
      "1217100    0\n",
      "539580     1\n",
      "Name: deltaval, dtype: int32\n",
      "91036\n"
     ]
    }
   ],
   "source": [
    "# hand calculation of proportion correct guesses in validation set\n",
    "\n",
    "dvalid[\"deltaval\"] = abs(dvalid.target - dvalid.predround)\n",
    "print(dvalid[\"deltaval\"][:10])\n",
    "print(dvalid[\"deltaval\"].sum())\n",
    "# print(\"percentage correct\")\n",
    "# print((len(deltaval) - deltaval.sum())/len(deltaval))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "917237     0\n",
      "346801     0\n",
      "513886     0\n",
      "289945     0\n",
      "358978     1\n",
      "2415560    0\n",
      "643834     0\n",
      "2263615    1\n",
      "108015     0\n",
      "464391     0\n",
      "Name: deltaval, dtype: int32\n",
      "75202\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\personal\\manning\\tf2_test\\notebooks\\try_tf2\\lib\\site-packages\\ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "# hand calculation of proportion correct guesses in validation set\n",
    "\n",
    "test[\"deltaval\"] = abs(test.target - test.predround)\n",
    "print(test[\"deltaval\"][:10])\n",
    "print(test[\"deltaval\"].sum())\n",
    "# print(\"percentage correct\")\n",
    "# print((len(deltaval) - deltaval.sum())/len(deltaval))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>count</th>\n",
       "      <th>Route</th>\n",
       "      <th>Direction</th>\n",
       "      <th>hour</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "      <th>day</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>target</th>\n",
       "      <th>predround</th>\n",
       "      <th>predict</th>\n",
       "      <th>deltaval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1364770</th>\n",
       "      <td>2016-10-09</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.542842</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539580</th>\n",
       "      <td>2015-02-05</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>751605</th>\n",
       "      <td>2015-07-12</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.520703</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2107344</th>\n",
       "      <td>2018-04-13</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.611880</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2325702</th>\n",
       "      <td>2018-09-22</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>677142</th>\n",
       "      <td>2015-05-18</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1800223</th>\n",
       "      <td>2017-08-28</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.539457</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799820</th>\n",
       "      <td>2015-08-17</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.524267</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2030957</th>\n",
       "      <td>2018-02-15</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.702900</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2656163</th>\n",
       "      <td>2019-05-25</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766252</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2117895</th>\n",
       "      <td>2018-04-21</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.731611</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65855</th>\n",
       "      <td>2014-02-18</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.769492</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456734</th>\n",
       "      <td>2014-12-05</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.677513</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1638512</th>\n",
       "      <td>2017-04-30</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>29</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766853</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1342048</th>\n",
       "      <td>2016-09-22</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.525085</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>731238</th>\n",
       "      <td>2015-06-27</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>26</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.805126</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2279955</th>\n",
       "      <td>2018-08-19</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.711842</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1772626</th>\n",
       "      <td>2017-08-08</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.609712</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94867</th>\n",
       "      <td>2014-03-12</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618343</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2825657</th>\n",
       "      <td>2019-09-28</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.743119</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Report Date  count  Route  Direction  hour  year  month  daym  day  \\\n",
       "1364770  2016-10-09      0      9          1     4     2      9     8    6   \n",
       "539580   2015-02-05      0      7          1     9     1      1     4    3   \n",
       "751605   2015-07-12      0     10          4     9     1      6    11    6   \n",
       "2107344  2018-04-13      0     10          4     9     4      3    12    4   \n",
       "2325702  2018-09-22      0      7          4    13     4      8    21    5   \n",
       "677142   2015-05-18      0      7          1    17     1      4    17    0   \n",
       "1800223  2017-08-28      0     11          4     6     3      7    27    0   \n",
       "799820   2015-08-17      0      8          1     2     1      7    16    0   \n",
       "2030957  2018-02-15      0     13          4    21     4      1    14    3   \n",
       "2656163  2019-05-25      0      9          1    14     5      4    24    5   \n",
       "2117895  2018-04-21      0      8          1    15     4      3    20    5   \n",
       "65855    2014-02-18      0     13          4     6     0      1    17    1   \n",
       "456734   2014-12-05      0     13          1    10     0     11     4    4   \n",
       "1638512  2017-04-30      0      9          4    21     3      3    29    6   \n",
       "1342048  2016-09-22      0     10          4    21     2      8    21    3   \n",
       "731238   2015-06-27      0      9          1     5     1      5    26    5   \n",
       "2279955  2018-08-19      0      8          1    12     4      7    18    6   \n",
       "1772626  2017-08-08      0      4          4     4     3      7     7    1   \n",
       "94867    2014-03-12      0      8          1    22     0      2    11    2   \n",
       "2825657  2019-09-28      0      9          1    17     5      8    27    5   \n",
       "\n",
       "         Min Delay  target  predround   predict  deltaval  \n",
       "1364770        0.0       0          1  0.542842         1  \n",
       "539580         0.0       0          1  1.000000         1  \n",
       "751605         0.0       0          1  0.520703         1  \n",
       "2107344        0.0       0          1  0.611880         1  \n",
       "2325702        0.0       0          1  1.000000         1  \n",
       "677142         0.0       0          1  1.000000         1  \n",
       "1800223        0.0       0          1  0.539457         1  \n",
       "799820         0.0       0          1  0.524267         1  \n",
       "2030957        0.0       0          1  0.702900         1  \n",
       "2656163        0.0       0          1  0.766252         1  \n",
       "2117895        0.0       0          1  0.731611         1  \n",
       "65855          0.0       0          1  0.769492         1  \n",
       "456734         0.0       0          1  0.677513         1  \n",
       "1638512        0.0       0          1  0.766853         1  \n",
       "1342048        0.0       0          1  0.525085         1  \n",
       "731238         0.0       0          1  0.805126         1  \n",
       "2279955        0.0       0          1  0.711842         1  \n",
       "1772626        0.0       0          1  0.609712         1  \n",
       "94867          0.0       0          1  0.618343         1  \n",
       "2825657        0.0       0          1  0.743119         1  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get subset of dataframe with wrong guesses\n",
    "# k1 = df.loc[(df.Product == p_id)\n",
    "dvalidwrong = dvalid.loc[(dvalid.deltaval == 1)]\n",
    "dvalidright = dvalid.loc[(dvalid.deltaval == 0)]\n",
    "dvalidwrong.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>count</th>\n",
       "      <th>Route</th>\n",
       "      <th>Direction</th>\n",
       "      <th>hour</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "      <th>day</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>target</th>\n",
       "      <th>predict</th>\n",
       "      <th>predround</th>\n",
       "      <th>deltaval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>358978</th>\n",
       "      <td>2014-09-24</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>23</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2263615</th>\n",
       "      <td>2018-08-07</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>837958</th>\n",
       "      <td>2015-09-14</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346903</th>\n",
       "      <td>2014-09-15</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.574514</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1835869</th>\n",
       "      <td>2017-09-24</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2803178</th>\n",
       "      <td>2019-09-11</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2493120</th>\n",
       "      <td>2019-01-24</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.557305</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281971</th>\n",
       "      <td>2014-07-29</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259635</th>\n",
       "      <td>2016-07-23</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.176757</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423874</th>\n",
       "      <td>2016-11-22</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>506147</th>\n",
       "      <td>2015-01-11</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.927466</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>976357</th>\n",
       "      <td>2015-12-26</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>25</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.752488</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2050223</th>\n",
       "      <td>2018-03-02</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.731574</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>915623</th>\n",
       "      <td>2015-11-11</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2409895</th>\n",
       "      <td>2018-11-23</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.575972</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2646307</th>\n",
       "      <td>2019-05-18</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2135384</th>\n",
       "      <td>2018-05-04</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.745008</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226268</th>\n",
       "      <td>2014-06-17</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.566198</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2655705</th>\n",
       "      <td>2019-05-25</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2495355</th>\n",
       "      <td>2019-01-26</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.975624</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Report Date  count  Route  Direction  hour  year  month  daym  day  \\\n",
       "358978   2014-09-24      0      7          1     1     0      8    23    2   \n",
       "2263615  2018-08-07      0      7          4     8     4      7     6    1   \n",
       "837958   2015-09-14      0     11          2    12     1      8    13    0   \n",
       "346903   2014-09-15      0      7          3    13     0      8    14    0   \n",
       "1835869  2017-09-24      0      4          4    19     3      8    23    6   \n",
       "2803178  2019-09-11      0     12          4    13     5      8    10    2   \n",
       "2493120  2019-01-24      0      9          4     7     5      0    23    3   \n",
       "281971   2014-07-29      0      4          4    19     0      6    28    1   \n",
       "1259635  2016-07-23      0      8          1    12     2      6    22    5   \n",
       "1423874  2016-11-22      0      7          1     0     2     10    21    1   \n",
       "506147   2015-01-11      0      9          4    15     1      0    10    6   \n",
       "976357   2015-12-26      0      9          1     1     1     11    25    5   \n",
       "2050223  2018-03-02      0      4          4     4     4      2     1    4   \n",
       "915623   2015-11-11      0      7          3     7     1     10    10    2   \n",
       "2409895  2018-11-23      0     12          1    20     4     10    22    4   \n",
       "2646307  2019-05-18      0      4          4    15     5      4    17    5   \n",
       "2135384  2018-05-04      0      7          3    14     4      4     3    4   \n",
       "226268   2014-06-17      0     13          1    19     0      5    16    1   \n",
       "2655705  2019-05-25      0      4          1    13     5      4    24    5   \n",
       "2495355  2019-01-26      0      4          4     0     5      0    25    5   \n",
       "\n",
       "         Min Delay  target   predict  predround  deltaval  \n",
       "358978         0.0       0  1.000000          1         1  \n",
       "2263615        0.0       0  1.000000          1         1  \n",
       "837958         3.0       1  0.000000          0         1  \n",
       "346903         0.0       0  0.574514          1         1  \n",
       "1835869        0.0       0  1.000000          1         1  \n",
       "2803178        6.0       1  0.000000          0         1  \n",
       "2493120        0.0       0  0.557305          1         1  \n",
       "281971         0.0       0  1.000000          1         1  \n",
       "1259635       12.0       1  0.176757          0         1  \n",
       "1423874        0.0       0  1.000000          1         1  \n",
       "506147         0.0       0  0.927466          1         1  \n",
       "976357         0.0       0  0.752488          1         1  \n",
       "2050223        0.0       0  0.731574          1         1  \n",
       "915623         0.0       0  1.000000          1         1  \n",
       "2409895        0.0       0  0.575972          1         1  \n",
       "2646307        0.0       0  1.000000          1         1  \n",
       "2135384        0.0       0  0.745008          1         1  \n",
       "226268         0.0       0  0.566198          1         1  \n",
       "2655705        0.0       0  1.000000          1         1  \n",
       "2495355        0.0       0  0.975624          1         1  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get subset of dataframe with wrong guesses\n",
    "# k1 = df.loc[(df.Product == p_id)\n",
    "testwrong = test.loc[(test.deltaval == 1)]\n",
    "testright = test.loc[(test.deltaval == 0)]\n",
    "testwrong.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1fa514e7cc8>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAUeElEQVR4nO3dbYyd5Z3f8e+vOEFuElgDyYhitqaFbZeEJru4DmraarZIQLIvSCSiOkXBu0vlbUqqrOQXS/KirIKQglSWCtqw9S4WD6IhiGRrqg1LXdhpulqenIiNeSjFDTQ4IFDWFsGpoBny74tzjXLsjq85noczDOf7kY7OOf9zX/d9/W1rfud+mNupKiRJOpa/ttoTkCS9vRkUkqQug0KS1GVQSJK6DApJUte61Z7AcjvttNNq06ZNix7/k5/8hPe85z3LN6E1YNJ6nrR+wZ4nxVJ6/s53vvOjqnr/fJ+944Ji06ZN7N27d9HjZ2ZmmJ6eXr4JrQGT1vOk9Qv2PCmW0nOS/32szzz0JEnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6nrH/Wb2Uu374Wv8xtV/MvbtvvCVXx/7NiVpFAvuUSQ5M8mfJXkmyVNJvtDqv5fkh0meaI9PDI35YpL9SZ5NcvFQ/fwk+9pnNyVJq5+Y5Out/miSTUNjtiV5rj22LWfzkqSFjbJHMQvsqKrvJnkf8J0ke9pnN1bVvxleOMm5wFbgg8DfAP5rkl+qqreAW4DtwCPAt4BLgPuBK4FDVXV2kq3A9cA/TXIKcA2wGai27fuq6tDS2pYkjWrBPYqqermqvttevw48A5zRGXIpcHdVvVlVzwP7gS1JTgdOqqqHa/Afdd8BfHJozO3t9b3AhW1v42JgT1UdbOGwh0G4SJLG5LjOUbRDQr8CPAp8DPh8kiuAvQz2Og4xCJFHhoYdaLWfttdH12nPLwJU1WyS14BTh+vzjBme13YGeypMTU0xMzNzPG0dYWo97DhvdtHjF2spc16qw4cPr+r2x23S+gV7nhQr1fPIQZHkvcA3gN+pqh8nuQW4lsEhoWuBG4DfAjLP8OrUWeSYnxeqdgI7ATZv3lxLubXwzXft5oZ94z/H/8Ll02Pf5pxJux3zpPUL9jwpVqrnkS6PTfIuBiFxV1V9E6CqXqmqt6rqZ8AfAlva4geAM4eGbwReavWN89SPGJNkHXAycLCzLknSmIxy1VOAW4Fnqur3h+qnDy32KeDJ9vo+YGu7kuks4Bzgsap6GXg9yQVtnVcAu4fGzF3RdBnwUDuP8QBwUZINSTYAF7WaJGlMRjnG8jHgs8C+JE+02peAzyT5CINDQS8Avw1QVU8luQd4msEVU1e1K54APgfcBqxncLXT/a1+K3Bnkv0M9iS2tnUdTHIt8Hhb7stVdXBxrUqSFmPBoKiqP2f+cwXf6oy5Drhunvpe4EPz1N8APn2Mde0Cdi00T0nSyvAWHpKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqWvBoEhyZpI/S/JMkqeSfKHVT0myJ8lz7XnD0JgvJtmf5NkkFw/Vz0+yr312U5K0+olJvt7qjybZNDRmW9vGc0m2LWfzkqSFjbJHMQvsqKpfBi4ArkpyLnA18GBVnQM82N7TPtsKfBC4BPhqkhPaum4BtgPntMclrX4lcKiqzgZuBK5v6zoFuAb4KLAFuGY4kCRJK2/BoKiql6vqu+3168AzwBnApcDtbbHbgU+215cCd1fVm1X1PLAf2JLkdOCkqnq4qgq446gxc+u6F7iw7W1cDOypqoNVdQjYw8/DRZI0BuuOZ+F2SOhXgEeBqap6GQZhkuQDbbEzgEeGhh1otZ+210fX58a82NY1m+Q14NTh+jxjhue1ncGeClNTU8zMzBxPW0eYWg87zptd9PjFWsqcl+rw4cOruv1xm7R+wZ4nxUr1PHJQJHkv8A3gd6rqx+30wryLzlOrTn2xY35eqNoJ7ATYvHlzTU9PH2tuC7r5rt3csO+48nNZvHD59Ni3OWdmZoal/JmtNZPWL9jzpFipnke66inJuxiExF1V9c1WfqUdTqI9v9rqB4Azh4ZvBF5q9Y3z1I8Yk2QdcDJwsLMuSdKYjHLVU4BbgWeq6veHProPmLsKaRuwe6i+tV3JdBaDk9aPtcNUrye5oK3ziqPGzK3rMuChdh7jAeCiJBvaSeyLWk2SNCajHGP5GPBZYF+SJ1rtS8BXgHuSXAn8APg0QFU9leQe4GkGV0xdVVVvtXGfA24D1gP3twcMgujOJPsZ7Elsbes6mORa4PG23Jer6uAie5UkLcKCQVFVf8785woALjzGmOuA6+ap7wU+NE/9DVrQzPPZLmDXQvOUJK0MfzNbktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpK4FgyLJriSvJnlyqPZ7SX6Y5In2+MTQZ19Msj/Js0kuHqqfn2Rf++ymJGn1E5N8vdUfTbJpaMy2JM+1x7blalqSNLpR9ihuAy6Zp35jVX2kPb4FkORcYCvwwTbmq0lOaMvfAmwHzmmPuXVeCRyqqrOBG4Hr27pOAa4BPgpsAa5JsuG4O5QkLcmCQVFV3wYOjri+S4G7q+rNqnoe2A9sSXI6cFJVPVxVBdwBfHJozO3t9b3AhW1v42JgT1UdrKpDwB7mDyxJ0gpat4Sxn09yBbAX2NF+mJ8BPDK0zIFW+2l7fXSd9vwiQFXNJnkNOHW4Ps+YIyTZzmBvhampKWZmZhbd1NR62HHe7KLHL9ZS5rxUhw8fXtXtj9uk9Qv2PClWqufFBsUtwLVAtecbgN8CMs+y1amzyDFHFqt2AjsBNm/eXNPT052p9918125u2LeU/FycFy6fHvs258zMzLCUP7O1ZtL6BXueFCvV86KueqqqV6rqrar6GfCHDM4hwOBb/5lDi24EXmr1jfPUjxiTZB1wMoNDXcdalyRpjBYVFO2cw5xPAXNXRN0HbG1XMp3F4KT1Y1X1MvB6kgva+YcrgN1DY+auaLoMeKidx3gAuCjJhnYS+6JWkySN0YLHWJJ8DZgGTktygMGVSNNJPsLgUNALwG8DVNVTSe4BngZmgauq6q22qs8xuIJqPXB/ewDcCtyZZD+DPYmtbV0Hk1wLPN6W+3JVjXpSXZK0TBYMiqr6zDzlWzvLXwdcN099L/CheepvAJ8+xrp2AbsWmqMkaeX4m9mSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKlrwaBIsivJq0meHKqdkmRPkufa84ahz76YZH+SZ5NcPFQ/P8m+9tlNSdLqJyb5eqs/mmTT0JhtbRvPJdm2XE1LkkY3yh7FbcAlR9WuBh6sqnOAB9t7kpwLbAU+2MZ8NckJbcwtwHbgnPaYW+eVwKGqOhu4Ebi+resU4Brgo8AW4JrhQJIkjceCQVFV3wYOHlW+FLi9vb4d+ORQ/e6qerOqngf2A1uSnA6cVFUPV1UBdxw1Zm5d9wIXtr2Ni4E9VXWwqg4Be/j/A0uStMIWe45iqqpeBmjPH2j1M4AXh5Y70GpntNdH148YU1WzwGvAqZ11SZLGaN0yry/z1KpTX+yYIzeabGdwWIupqSlmZmYWnOixTK2HHefNLnr8Yi1lzkt1+PDhVd3+uE1av2DPk2Klel5sULyS5PSqerkdVnq11Q8AZw4ttxF4qdU3zlMfHnMgyTrgZAaHug4A00eNmZlvMlW1E9gJsHnz5pqenp5vsZHcfNdubti33Pm5sBcunx77NufMzMywlD+ztWbS+gV7nhQr1fNiDz3dB8xdhbQN2D1U39quZDqLwUnrx9rhqdeTXNDOP1xx1Ji5dV0GPNTOYzwAXJRkQzuJfVGrSZLGaMGvzkm+xuCb/WlJDjC4EukrwD1JrgR+AHwaoKqeSnIP8DQwC1xVVW+1VX2OwRVU64H72wPgVuDOJPsZ7Elsbes6mORa4PG23Jer6uiT6pKkFbZgUFTVZ47x0YXHWP464Lp56nuBD81Tf4MWNPN8tgvYtdAcJUkrx9/MliR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKlrSUGR5IUk+5I8kWRvq52SZE+S59rzhqHlv5hkf5Jnk1w8VD+/rWd/kpuSpNVPTPL1Vn80yaalzFeSdPyWY4/i16rqI1W1ub2/Gniwqs4BHmzvSXIusBX4IHAJ8NUkJ7QxtwDbgXPa45JWvxI4VFVnAzcC1y/DfCVJx2ElDj1dCtzeXt8OfHKofndVvVlVzwP7gS1JTgdOqqqHq6qAO44aM7eue4EL5/Y2JEnjsW6J4wv4L0kK+A9VtROYqqqXAarq5SQfaMueATwyNPZAq/20vT66Pjfmxbau2SSvAacCPxqeRJLtDPZImJqaYmZmZtENTa2HHefNLnr8Yi1lzkt1+PDhVd3+uE1av2DPk2Klel5qUHysql5qYbAnyf/oLDvfnkB16r0xRxYGAbUTYPPmzTU9Pd2ddM/Nd+3mhn1L/WM5fi9cPj32bc6ZmZlhKX9ma82k9Qv2PClWquclHXqqqpfa86vAHwNbgFfa4STa86tt8QPAmUPDNwIvtfrGeepHjEmyDjgZOLiUOUuSjs+igyLJe5K8b+41cBHwJHAfsK0ttg3Y3V7fB2xtVzKdxeCk9WPtMNXrSS5o5x+uOGrM3LouAx5q5zEkSWOylGMsU8Aft3PL64D/WFV/muRx4J4kVwI/AD4NUFVPJbkHeBqYBa6qqrfauj4H3AasB+5vD4BbgTuT7GewJ7F1CfOVJC3CooOiqr4PfHie+l8BFx5jzHXAdfPU9wIfmqf+Bi1oJEmrY/xnbaVm09V/sirb3XHeLL+xStt+4Su/virblZbCW3hIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV3eFFDSilqtmz/edsl7VmW770TuUUiSugwKSVKXQSFJ6jIoJEldBoUkqcurniRpmb3TrvQyKMS+H762av+HtKS3P4PibWK1voEA7Dhv1TY9cVbr73nHebN+GdCiGRSS3pHcU14+nsyWJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1rYmgSHJJkmeT7E9y9WrPR5Imyds+KJKcAPx74OPAucBnkpy7urOSpMnxtg8KYAuwv6q+X1X/F7gbuHSV5yRJEyNVtdpz6EpyGXBJVf3z9v6zwEer6vNDy2wHtre3fwd4dgmbPA340RLGr0WT1vOk9Qv2PCmW0vPfrKr3z/fBWrh7bOapHZFuVbUT2LksG0v2VtXm5VjXWjFpPU9av2DPk2Klel4Lh54OAGcOvd8IvLRKc5GkibMWguJx4JwkZyV5N7AVuG+V5yRJE+Ntf+ipqmaTfB54ADgB2FVVT63gJpflENYaM2k9T1q/YM+TYkV6ftufzJYkra61cOhJkrSKDApJUtdEBsVCtwTJwE3t8+8l+dXVmOdyGqHny1uv30vyF0k+vBrzXE6j3volyd9P8lb7nZ01bZSek0wneSLJU0n+27jnuNxG+Ld9cpL/nOQvW8+/uRrzXC5JdiV5NcmTx/h8+X9+VdVEPRicEP9fwN8C3g38JXDuUct8Arifwe9wXAA8utrzHkPP/wDY0F5/fBJ6HlruIeBbwGWrPe8x/D3/AvA08Ivt/QdWe95j6PlLwPXt9fuBg8C7V3vuS+j5HwO/Cjx5jM+X/efXJO5RjHJLkEuBO2rgEeAXkpw+7okuowV7rqq/qKpD7e0jDH5fZS0b9dYv/wr4BvDqOCe3Qkbp+Z8B36yqHwBU1Vrve5SeC3hfkgDvZRAUs+Od5vKpqm8z6OFYlv3n1yQGxRnAi0PvD7Ta8S6zlhxvP1cy+Eayli3Yc5IzgE8BfzDGea2kUf6efwnYkGQmyXeSXDG22a2MUXr+d8AvM/hF3X3AF6rqZ+OZ3qpY9p9fb/vfo1gBC94SZMRl1pKR+0nyawyC4h+u6IxW3ig9/1vgd6vqrcGXzTVvlJ7XAecDFwLrgYeTPFJV/3OlJ7dCRun5YuAJ4J8AfxvYk+S/V9WPV3pyq2TZf35NYlCMckuQd9ptQ0bqJ8nfA/4I+HhV/dWY5rZSRul5M3B3C4nTgE8kma2q/zSeKS67Uf9t/6iqfgL8JMm3gQ8DazUoRun5N4Gv1OAA/v4kzwN/F3hsPFMcu2X/+TWJh55GuSXIfcAV7eqBC4DXqurlcU90GS3Yc5JfBL4JfHYNf7sctmDPVXVWVW2qqk3AvcC/XMMhAaP9294N/KMk65L8deCjwDNjnudyGqXnHzDYgyLJFIM7TH9/rLMcr2X/+TVxexR1jFuCJPkX7fM/YHAFzCeA/cD/YfCNZM0ased/DZwKfLV9w56tNXznzRF7fkcZpeeqeibJnwLfA34G/FFVzXuZ5Vow4t/ztcBtSfYxOCzzu1W1Zm8/nuRrwDRwWpIDwDXAu2Dlfn55Cw9JUtckHnqSJB0Hg0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSp6/8BgCwHgTe8ZHkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dvalid.hist(range = (0,5))\n",
    "# apar_ds.Time_to_relief.hist(range = (0,5))\n",
    "dvalid.predict.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1fa515551c8>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAASiUlEQVR4nO3df6zd9X3f8eerOMm8tCEGwhXCdGbD20qSpS2eQes23cUddpNpZBKRvLHgVp6sZXTKJKQV+sesBSHBH1k6spLKSixMhkosmtZeO4Ys2F02ld9bEgcYwysMXKygxB7FTGExfe+P87nj4Fx/7vH9cS6X83xIR/ec9/f7+X4/72vrvu73xzk3VYUkSWfyEys9AUnSO5tBIUnqMigkSV0GhSSpy6CQJHWtWekJLLULLrigNmzYsODxr7/+Ou9///uXbkKrwKT1PGn9gj1PisX0/OSTT36/qj4017J3XVBs2LCBJ554YsHjZ2ZmmJ6eXroJrQKT1vOk9Qv2PCkW03OS/3WmZZ56kiR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdb3r3pm9WIf/+FV++aY/GPt+X7jtk2PfpySNwiMKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1DVyUCQ5J8l/S/L77fV5SQ4lea59XTe07s1JjiR5NsnWofoVSQ63ZXckSau/L8nXW/3RJBuGxuxo+3guyY6laFqSNLqzOaL4HPDM0OubgAeraiPwYHtNksuB7cCHgW3AnUnOaWO+DOwCNrbHtlbfCZyoqsuALwK3t22dB+wGrgQ2A7uHA0mStPxGCook64FPAl8ZKl8D7GvP9wGfGqrfW1VvVNXzwBFgc5KLgA9U1cNVVcDdp42Z3dZ9wJZ2tLEVOFRVx6vqBHCIt8JFkjQGo/4p1N8A/jnwU0O1qao6BlBVx5Jc2OoXA48MrXe01X7Unp9enx3zUtvWqSSvAucP1+cY8/8l2cXgSIWpqSlmZmZGbOvHTa2FGz96asHjF2oxc16skydPruj+x23S+gV7nhTL1fO8QZHk7wCvVNWTSaZH2GbmqFWnvtAxbxWq9gB7ADZt2lTT06NMc25fuucAXzg8/j8l/sJ102Pf56yZmRkW8z1bbSatX7DnSbFcPY9y6ukXgL+b5AXgXuDjSf4t8L12Oon29ZW2/lHgkqHx64GXW339HPW3jUmyBjgXON7ZliRpTOYNiqq6uarWV9UGBhepH6qqfwgcBGbvQtoBHGjPDwLb251MlzK4aP1YO031WpKr2vWH608bM7uta9s+CngAuDrJunYR++pWkySNyWLOsdwG7E+yE3gR+DRAVT2VZD/wNHAKuKGq3mxjPgvcBawF7m8PgK8CX0tyhMGRxPa2reNJbgEeb+t9vqqOL2LOkqSzdFZBUVUzwEx7/gNgyxnWuxW4dY76E8BH5qj/kBY0cyzbC+w9m3lKkpaO78yWJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeqaNyiS/JkkjyX5dpKnkvzLVj8vyaEkz7Wv64bG3JzkSJJnk2wdql+R5HBbdkeStPr7kny91R9NsmFozI62j+eS7FjK5iVJ8xvliOIN4ONV9THgZ4FtSa4CbgIerKqNwIPtNUkuB7YDHwa2AXcmOadt68vALmBje2xr9Z3Aiaq6DPgicHvb1nnAbuBKYDOweziQJEnLb96gqIGT7eV72qOAa4B9rb4P+FR7fg1wb1W9UVXPA0eAzUkuAj5QVQ9XVQF3nzZmdlv3AVva0cZW4FBVHa+qE8Ah3goXSdIYrBllpXZE8CRwGfCbVfVokqmqOgZQVceSXNhWvxh4ZGj40Vb7UXt+en12zEttW6eSvAqcP1yfY8zw/HYxOFJhamqKmZmZUdqa09RauPGjpxY8fqEWM+fFOnny5Iruf9wmrV+w50mxXD2PFBRV9Sbws0k+CPxuko90Vs9cm+jUFzpmeH57gD0AmzZtqunp6c70+r50zwG+cHikb8uSeuG66bHvc9bMzAyL+Z6tNpPWL9jzpFiuns/qrqeq+t/ADIPTP99rp5NoX19pqx0FLhkath54udXXz1F/25gka4BzgeOdbUmSxmSUu54+1I4kSLIW+EXgvwMHgdm7kHYAB9rzg8D2difTpQwuWj/WTlO9luSqdv3h+tPGzG7rWuChdh3jAeDqJOvaReyrW02SNCajnGO5CNjXrlP8BLC/qn4/ycPA/iQ7gReBTwNU1VNJ9gNPA6eAG9qpK4DPAncBa4H72wPgq8DXkhxhcCSxvW3reJJbgMfbep+vquOLaViSdHbmDYqq+g7wc3PUfwBsOcOYW4Fb56g/AfzY9Y2q+iEtaOZYthfYO988JUnLw3dmS5K6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1zRsUSS5J8h+TPJPkqSSfa/XzkhxK8lz7um5ozM1JjiR5NsnWofoVSQ63ZXckSau/L8nXW/3RJBuGxuxo+3guyY6lbF6SNL9RjihOATdW1c8AVwE3JLkcuAl4sKo2Ag+217Rl24EPA9uAO5Oc07b1ZWAXsLE9trX6TuBEVV0GfBG4vW3rPGA3cCWwGdg9HEiSpOU3b1BU1bGq+q/t+WvAM8DFwDXAvrbaPuBT7fk1wL1V9UZVPQ8cATYnuQj4QFU9XFUF3H3amNlt3QdsaUcbW4FDVXW8qk4Ah3grXCRJY7DmbFZup4R+DngUmKqqYzAIkyQXttUuBh4ZGna01X7Unp9enx3zUtvWqSSvAucP1+cYMzyvXQyOVJiammJmZuZs2nqbqbVw40dPLXj8Qi1mzot18uTJFd3/uE1av2DPk2K5eh45KJL8JPA7wD+rqj9plxfmXHWOWnXqCx3zVqFqD7AHYNOmTTU9PX2muc3rS/cc4AuHzyo/l8QL102PfZ+zZmZmWMz3bLWZtH7BnifFcvU80l1PSd7DICTuqapvtPL32ukk2tdXWv0ocMnQ8PXAy62+fo7628YkWQOcCxzvbEuSNCaj3PUU4KvAM1X1r4YWHQRm70LaARwYqm9vdzJdyuCi9WPtNNVrSa5q27z+tDGz27oWeKhdx3gAuDrJunYR++pWkySNySjnWH4B+AxwOMm3Wu3XgduA/Ul2Ai8CnwaoqqeS7AeeZnDH1A1V9WYb91ngLmAtcH97wCCIvpbkCIMjie1tW8eT3AI83tb7fFUdX2CvkqQFmDcoquq/MPe1AoAtZxhzK3DrHPUngI/MUf8hLWjmWLYX2DvfPCVJy8N3ZkuSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdc0bFEn2JnklyXeHauclOZTkufZ13dCym5McSfJskq1D9SuSHG7L7kiSVn9fkq+3+qNJNgyN2dH28VySHUvVtCRpdKMcUdwFbDutdhPwYFVtBB5sr0lyObAd+HAbc2eSc9qYLwO7gI3tMbvNncCJqroM+CJwe9vWecBu4EpgM7B7OJAkSeMxb1BU1TeB46eVrwH2tef7gE8N1e+tqjeq6nngCLA5yUXAB6rq4aoq4O7Txsxu6z5gSzva2AocqqrjVXUCOMSPB5YkaZmtWeC4qao6BlBVx5Jc2OoXA48MrXe01X7Unp9enx3zUtvWqSSvAucP1+cY8zZJdjE4WmFqaoqZmZkFtgVTa+HGj55a8PiFWsycF+vkyZMruv9xm7R+wZ4nxXL1vNCgOJPMUatOfaFj3l6s2gPsAdi0aVNNT0/PO9Ez+dI9B/jC4aX+tszvheumx77PWTMzMyzme7baTFq/YM+TYrl6XuhdT99rp5NoX19p9aPAJUPrrQdebvX1c9TfNibJGuBcBqe6zrQtSdIYLTQoDgKzdyHtAA4M1be3O5kuZXDR+rF2muq1JFe16w/XnzZmdlvXAg+16xgPAFcnWdcuYl/dapKkMZr3HEuS3wamgQuSHGVwJ9JtwP4kO4EXgU8DVNVTSfYDTwOngBuq6s22qc8yuINqLXB/ewB8FfhakiMMjiS2t20dT3IL8Hhb7/NVdfpFdUnSMps3KKrq759h0ZYzrH8rcOsc9SeAj8xR/yEtaOZYthfYO98cJUnLx3dmS5K6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklS17x/ClWSdHY23PQHK7Lfu7a9f1m26xGFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktTlhwK+Q6zUh4jB8n2QmKR3B4NCHP7jV/nlFQiqF2775Nj3KensGRRaMe+2j2KW3q0MCk2clTqCAo+itDp5MVuS1OURhTRGnm7TamRQSBPA021ajFURFEm2Af8aOAf4SlXdtsJTkjQij6JWv3d8UCQ5B/hN4G8DR4HHkxysqqdXdmaS3slW8ijq3WY1XMzeDBypqj+qqv8L3Atcs8JzkqSJkapa6Tl0JbkW2FZV/6i9/gxwZVX96tA6u4Bd7eVfAp5dxC4vAL6/iPGr0aT1PGn9gj1PisX0/Oeq6kNzLXjHn3oCMkftbelWVXuAPUuys+SJqtq0FNtaLSat50nrF+x5UixXz6vh1NNR4JKh1+uBl1doLpI0cVZDUDwObExyaZL3AtuBgys8J0maGO/4U09VdSrJrwIPMLg9dm9VPbWMu1ySU1irzKT1PGn9gj1PimXp+R1/MVuStLJWw6knSdIKMigkSV0TGRRJtiV5NsmRJDfNsTxJ7mjLv5Pk51dinktphJ6va71+J8kfJvnYSsxzKc3X89B6fzXJm+09O6vaKD0nmU7yrSRPJflP457jUhvh//a5Sf5dkm+3nn9lJea5VJLsTfJKku+eYfnS//yqqol6MLgg/j+BPw+8F/g2cPlp63wCuJ/BeziuAh5d6XmPoee/Bqxrz39pEnoeWu8h4N8D1670vMfw7/xB4Gngp9vrC1d63mPo+deB29vzDwHHgfeu9NwX0fPfBH4e+O4Zli/5z69JPKIY5SNBrgHuroFHgA8muWjcE11C8/ZcVX9YVSfay0cYvF9lNRv1o1/+KfA7wCvjnNwyGaXnfwB8o6peBKiq1d73KD0X8FNJAvwkg6A4Nd5pLp2q+iaDHs5kyX9+TWJQXAy8NPT6aKud7Tqrydn2s5PBbySr2bw9J7kY+HvAb41xXstplH/nvwisSzKT5Mkk149tdstjlJ7/DfAzDN6oexj4XFX96XimtyKW/OfXO/59FMtg3o8EGXGd1WTkfpL8LQZB8deXdUbLb5SefwP4tap6c/DL5qo3Ss9rgCuALcBa4OEkj1TV/1juyS2TUXreCnwL+DjwF4BDSf5zVf3Jck9uhSz5z69JDIpRPhLk3faxISP1k+SvAF8BfqmqfjCmuS2XUXreBNzbQuIC4BNJTlXV741niktu1P/b36+q14HXk3wT+BiwWoNilJ5/BbitBifwjyR5HvjLwGPjmeLYLfnPr0k89TTKR4IcBK5vdw9cBbxaVcfGPdElNG/PSX4a+AbwmVX82+WweXuuqkurakNVbQDuA/7JKg4JGO3/9gHgbyRZk+TPAlcCz4x5nktplJ5fZHAERZIpBp8w/UdjneV4LfnPr4k7oqgzfCRIkn/clv8WgztgPgEcAf4Pg99IVq0Re/4XwPnAne037FO1ij95c8Se31VG6bmqnknyH4DvAH/K4C9Gznmb5Wow4r/zLcBdSQ4zOC3za1W1aj9+PMlvA9PABUmOAruB98Dy/fzyIzwkSV2TeOpJknQWDApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkrv8HP/0fDcJ0ZYIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dvalid.hist(range = (0,5))\n",
    "# apar_ds.Time_to_relief.hist(range = (0,5))\n",
    "test.predict.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19370\n",
      "percentage correct validate\n",
      "0.8063\n"
     ]
    }
   ],
   "source": [
    "# get delta between predictions on training set and actual training target values\n",
    "# hand calculate accuracy on training set as ratio of (total training samples - wrong training predictions)/total training samples\n",
    "\n",
    "deltatr = abs(dvalid.target[:100000] - dvalid.predround[:100000])\n",
    "deltatr[:50]\n",
    "print(deltatr.sum())\n",
    "print(\"percentage correct validate\")\n",
    "print((len(deltatr) - deltatr.sum())/len(deltatr))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['loss', 'accuracy', 'accuracy_1', 'val_loss', 'val_accuracy', 'val_accuracy_1'])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXzcdbX4/9eZmezJpE3Tfd9bWtpCS9n3xYoFRFFBRMWriMJVrrhwvfd6uXqv93cfqD/vFRS5yHVDFlmkQGWVraxNoelemqZbmrTNnsxkmSRzvn/MZ9JpOkk+SWeyzXk+Hnkk81nfnzaZM+/tvEVVMcYYY7ryDHYBjDHGDE0WIIwxxsRlAcIYY0xcFiCMMcbEZQHCGGNMXBYgjDHGxGUBwhhARH4rIv/u8ti9InJJsstkzGCzAGGMMSYuCxDGjCAi4hvsMpiRwwKEGTacpp3viMgmEQmKyG9EZLyI/FVEGkXkJREZHXP8lSKyVUTqRORVEVkYs+8UEXnfOe8RILPLvVaLyEbn3LdEZInLMn5MRD4QkQYROSAid3bZf45zvTpn/xed7Vki8lMR2Sci9SKyztl2gYiUxfl3uMT5+U4ReUxE/igiDcAXRWSliLzt3KNCRO4WkfSY8xeJyIsiUiMih0Xk+yIyQUSaRGRMzHHLRaRSRNLcPLsZeSxAmOHmk8ClwDzgCuCvwPeBQiK/z98AEJF5wEPAbcBYYC3wtIikO2+WfwH+ABQAf3aui3PuqcADwFeBMcCvgTUikuGifEHg88Ao4GPA10Tk4851pznl/YVTpmXARue8nwDLgbOcMn0XCLv8N7kKeMy554NAB/APzr/JmcDFwNedMuQBLwHPAZOAOcDLqnoIeBX4dMx1Pwc8rKptLsthRhgLEGa4+YWqHlbVg8AbwLuq+oGqtgJPAqc4x30GeFZVX3Te4H4CZBF5Az4DSAN+rqptqvoYsD7mHl8Bfq2q76pqh6r+Dmh1zuuRqr6qqptVNayqm4gEqfOd3dcDL6nqQ859q1V1o4h4gC8B31TVg84933KeyY23VfUvzj2bVXWDqr6jqu2qupdIgIuWYTVwSFV/qqotqtqoqu86+35HJCggIl7gOiJB1KQoCxBmuDkc83NznNe5zs+TgH3RHaoaBg4Ak519B/XYTJX7Yn6eDtzuNNHUiUgdMNU5r0cicrqIvOI0zdQDNxP5JI9zjd1xTisk0sQVb58bB7qUYZ6IPCMih5xmpx+7KAPAU8BJIjKLSC2tXlXf62eZzAhgAcKMVOVE3ugBEBEh8uZ4EKgAJjvboqbF/HwA+A9VHRXzla2qD7m475+ANcBUVc0H7gWi9zkAzI5zThXQ0s2+IJAd8xxeIs1TsbqmZP4VsAOYq6p+Ik1wvZUBVW0BHiVS07kBqz2kPAsQZqR6FPiYiFzsdLLeTqSZ6C3gbaAd+IaI+ETkE8DKmHP/F7jZqQ2IiOQ4nc95Lu6bB9SoaouIrAQ+G7PvQeASEfm0c98xIrLMqd08APxMRCaJiFdEznT6PD4EMp37pwH/DPTWF5IHNAABEVkAfC1m3zPABBG5TUQyRCRPRE6P2f974IvAlcAfXTyvGcEsQJgRSVV3EmlP/wWRT+hXAFeoakhVQ8AniLwR1hLpr3gi5twiIv0Qdzv7S5xj3fg68EMRaQR+QCRQRa+7H7icSLCqIdJBvdTZ/W1gM5G+kBrgvwCPqtY717yfSO0nCBwzqimObxMJTI1Egt0jMWVoJNJ8dAVwCNgFXBiz/00inePvO/0XJoWJLRhkjIklIn8D/qSq9w92WczgsgBhjOkkIqcBLxLpQ2kc7PKYwWVNTMYYAETkd0TmSNxmwcGA1SCMMcZ0w2oQxhhj4hpRib0KCwt1xowZg10MY4wZNjZs2FClql3n1gAjLEDMmDGDoqKiwS6GMcYMGyKyr7t91sRkjDEmLgsQxhhj4rIAYYwxJi4LEMYYY+KyAGGMMSYuCxDGGGPisgBhjDEmLgsQxphBE2xt59H1B7CUP0OTBQhjzKB5uric7z6+iff21Ax2UUwcFiCMSXGby+q56fdFhNrDA37v3ZUBANbvtQAxFFmAMCbF3b+ulBe2HWZ/TXDA772nKnLPd60GMSRZgDAmhTWHOnhx22EAjjS0Dvj9SysjAWLDvlraOga+BmN6ZgHCmBT20vbDNIU6ADjSOLABoq0jzP6aJmaMyaYp1MHW8oYBvb/pnQUIY1LYmuJyRmWnAXCkseWEr7e/uomqgLtAc6Cmifaw8pnTpgHw3p7qE76/SSwLEMakqPrmNl7bWcknTplCVpqXw/1sYgqHlb/tOMwNv3mX8+56hX94ZKOr86LNS6fPKmBmYY6NZBqCRtR6EMYY957fcohQR5grl03i5R2H+9zE1NDSxp+Lyvj923vZV93EeH8GCybksamsHlVFRHo8P9pBPaswh9NnFrB2cwXhsOLx9HyeGThJrUGIyCoR2SkiJSJyR5z9+SLytIgUi8hWEbkxZt9eEdksIhtFxFYBMibB1hSXM31MNkun5DMuL4MjDe6bmO55pYQzfvwyP3pmG4W5GfziulNY972L+Ozp06hvbnNVGymtClCQk86o7HRWziygoaWdnYcbT+SRTIIlrQYhIl7gHuBSoAxYLyJrVHVbzGG3ANtU9QoRGQvsFJEHVTXk7L9QVauSVUZjUlVlYytv7a7i6xfMQUQYl5fJ9gp3ncQtbR3c9fxOzpw1hu9fvpCTp+R37ps/Pg+A7YcamJCf2eN1SiuDzCrMAWDlzAIA3ttTw8KJ/v48kkmCZNYgVgIlqlrqvOE/DFzV5RgF8iRSF80FaoD2JJbJGAOR5hyFK5dNAmBsXobrJqbDTk3jk8unHBMcABZMiLy57zzUe02gtCrITCdATBmdzeRRWdYPMcQkM0BMBg7EvC5ztsW6G1gIlAObgW+qanQwtAIviMgGEbkpieU0JuWsKS5nwYQ85jmf+Mf5Mwi0ttMU6v3zWUV9JEBMjFNDyM9OY2J+Jjt6qY00trRR2djKrLG5ndtWzizg3T01lpdpCElmgIjX09T1f/4jwEZgErAMuFtEovXLs1X1VOCjwC0icl7cm4jcJCJFIlJUWVmZoKIbM3KV1TaxYV8tVyyd1LltXF7kzd7NZLmK+maAbpuQFkzIY0cvNYhoB3W0BgGRAFEVaO3cZwZfMgNEGTA15vUUIjWFWDcCT2hECbAHWACgquXO9yPAk0SarI6jqvep6gpVXTF27NgEP4IxI8/TxRUAXLEkNkBkAO4my/VUgwCYP8HP7spAj7mdokNcZ489NkAA/W5meuL9Mv7w9t5+nWviS2aAWA/MFZGZIpIOXAus6XLMfuBiABEZD8wHSkUkR0TynO05wGXAliSW1ZiUsaa4nGVTRzFtTHbntnH+aIDofSTTofoW8rPSyE6PP8Zl4cQ82jqU0qpAt9corQzgEY4pw6zCHApz0/sVIEorA3zv8U3c+fQ29lVbDSRRkhYgVLUduBV4HtgOPKqqW0XkZhG52TnsR8BZIrIZeBn4njNqaTywTkSKgfeAZ1X1uWSV1ZhUUXKkke0VDVwZ07wEML4PTUzldS3d1h4A5k+I9Gv01FFdWhVkyuhsMnzezm0i0tkP0Reqyp1PbyPT58XrEe55paRP55vuJXWinKquBdZ22XZvzM/lRGoHXc8rBZYms2zGpKI1xRV4BFYvmXjM9lHZaaR7Pa6amA41NPcYIGYV5pLmFbZXNHLVsvjHlFYGmRXTvBS1ckYBazcfoqy2iSmjs+OcebwXtx3m9Q8r+cHqk9hf08Qf39nH3180l6kF7s433bNUG8akCFXl6eJyzpg1hnH+Y9/gRSQy1NXFZLlD9S1MyM/qdn+6z8PssbnsPBR/JJOqsqcqyKzC3OP2rZw5BnC/PkRLWwc/fGYb88fn8fkzp3Pz+bPxiNUiEsUChDEpYsvBBvZUBY9rXopyMxeitb2DqkCoxxoE9DyS6VBDC81tHcyMU4OYPyEPf6bPdT/Er17dTVltM3deuQif18OE/EyuXTmVxzaUUVbb5OoaJ6qivpkv/66IK36xbsTN47AAYUyKeHZzBWleYdXiCXH3j8vL6LWT+nB9JID0Nkt6/gQ/FfUt1De1HbevcwRT4fEBwusRTpvhrh9if3UTv3ptN1csncSZs8d0br/5/NmIwC9f3d3rNU6EqvLwe/u57Gev82ZJFVWBVj7967e5/dFi1xlthzoLEMakiL1VQWaMyWFUdnrc/eP8vdcgonMgJvXQxASwYGKko3pHnGam0ugciDg1CIhkdy2tDFLZS1l++Mw2fB7hny5feMz2SaOy+PSKqfy56AAH65p7vEZ/Hahp4obfvMcdT2xm0WQ/z912Li/ffj5fv2A2a4oPctFPXuUPb++lIzy8J/1ZgDAmRVQHWxmTGz84QGSyXF1TG63tHd0eE50D0VsNYkF0JFOc5HullQGy071M8Me/hpt+iFd2HOGl7Yf5xsVz45bl6xfOAeBXr3bfF1G0t4a/f+gDSo64TxAYDit/eHsvq37+Oh/sr+XfP76YP335DKaPySE73cd3Vy3gr988j8WT8/mXp7by8XveZOOBOtfXH2osQBiTIqoDIcbkZnS7PzpZrqdP7r1Nkoua4M8kPyuN7RXHv/nucXIwdZcOfNEkP9np3m7b81vbO/i3p7cyqzCHL509M+4xk0dlcc3yqTy6vqyz1hPVEVbu/tsuPnPfOzxdXM4nfvkWb5X0nhO0or6Z6+9/l395aiunTh/NC986n8+dMf249ORzxuXy4JdP53+uO4XDDS18/J43+cID7/FWSdWwSyNiAcKYFFEVaKUwp4cahL/32dSH6pvxZ/rIyeh5hLyIMH9CXtyRTKWVwWNSbHSV5vWwfProuP0QLW0d/M/Lu9hb3cSdVy4i3df9W9jXL5hNWJV7Y/oiDje08Ln73+UnL3zI6iUTee62c5mQn8nnH3iPR4sOdHutv26uYNXP32BTWR3/9cmT+f2XVjJ5VPfNbCLClUsn8fLt5/Odj8xna3kDn73/Xa68+02eLi6nfZisv20LBhmTAkLtYRpa2nupQfQ+Wa68voWJvfQ/RC2ckMdjG8qOWQSotb2DstomPn5K17ydx1o5o4CfvfQhOw81UnIkwIZ9tWzYX8vWg/W0h5XLT57AefN6Tq0ztSCbT546hYfWH+DrF85hW3kDt/+5mOZQB3dds4Rrlk9BRHjsa2dxy4Pv893HNrGvOsjtl87vLG+wtZ0fPr2NR4oOsHRKPv997SnM6CG4dZWXmcYtF87h786ZyZMfHOR/Xy/l7x/6gKkFWdxwxnTys9IItnbQFGon4HxvaevgS+fM7MyMO5gsQBiTAmqCkSVWeuyD8EebmLofyRSZA9Fz81LU/Al+gqEODtY1d05a21/dRFiPzcEUz8qZBajCR37+OgCZaR6WThnFTefN4tRpo3sNDlG3XDiHx94v4/r736XkSICFE/384rpTmDPu6BwMf2YaD3zxNH7w1BbueWU3+6qb+MmnlrLrcIBvPPwBe6uD3HLhbG67ZB5p3v41umSmeblu5TQ+s2IqL20/zK9fL+XHa3ccc0y610N2hpdASzvNbWF+cd0p/bpXIlmAMCYFRIddjsnpvgYxJicDj/TcxFRR38Liye4+2UZHMm2vaOgMELsrj8/iGs/y6aP5xkVzGJ2TzvLpo1k40d+vN+dpY7L55KmTebSojC+eNYM7PrqAzDTvcceleT38+OqTmTEmh//86w52HGpkX3WQwtwMHvrKGZwxa0ycq/edxyNctmgCly2awIGaJjweISfdS3a6r7O57PtPbuYvHxykOdRBVvrxZR1IFiCMSQHVTg2isIcahNcjFOZmdNvEFJkk18oEv7smpuhaEzsPNXLZosjci3hpvuPxeT1867L5ru7Tmx9etZgbz57Z60p1IsJXz5/NtIJsvvVoMZeeNJ4fX31yt8OCT1R3qUBWnzyRP727n1d2HuHykyfGPWagWIAwJgVUR2sQPfRBQKSZ6XA3TUzRwNHbCKao3Awf0wqyj5lRXVoZYGxeBnmZaa6ukQiZad4+LWP60ZMncvHC8T12gCfTypkFFOam8+ymikEPEDaKyZgUUB3ovQ8CIh3V3dUg3M6BiDV/Qt4xk+VKq46uQz2UDVZwgEjtadXiCby847CrFf6SyQKEMSmgKthKutdDXi/DU8f1kI+pcxb1KPcBYuGEPPZUBWlpi0y+21MVPGaZURPf6iWTaGkL8/L2I4NaDgsQxqSAyCS59G4np0WNy8ugOtgad5z+0RqEuz4IgAUT/YQVSo4EqGsKURMMDYsaxGA7bUYBY/MyeHZTxaCWwwKEMSmgOtBzmo2osf5MVI92asc6VN9CXoaP3F5qIbGiiwdtr2joHMEUbx0IcyyvR7h88QRe2XmEQGvPzUzhsCZthrYFCGNSQHUw1OMQ16jOtanj9EOU1zUzsQ/NSwAzxuSQ4fOw81Bj5wgma2JyZ/XSSbS2h3l5++Eej7t/XSk3/Oa9pPRXWIAwJgVEm5h6M95JoBcv7fehhp4XCorH6xHmjY+sDVFaGcDnEaaM7ts1UtXyaaMZ78/gmR6amfZWBfnpCx+Sle4lK878jhOV1AAhIqtEZKeIlIjIHXH254vI0yJSLCJbReRGt+caY9xR1Ugepl6GuEJMDSJOR3VFfQsTu8nA2pPo4kF7qoJMG5Pd79nIqcbjES4/eSKv7aykseX4dTVUlTue2ES618OPrlrca/9Sv8qQ8Cs6RMQL3AN8FDgJuE5ETupy2C3ANlVdClwA/FRE0l2ea4xxIRjqoLU9zJgeEvVFRYNI1yamUHuYqkBrn5uYINIPURVopWhfrXVQ99HqJRMJdYR5KU4z08PrD/BOaQ3f/9jCPg097otkhvKVQImqlqpqCHgYuKrLMQrkSST05QI1QLvLc40xLridJAeR8f8FOenHNTEdbmhB1f0kuVjRSWqVja3W/9BHp0wdzaT8TJ4pPraZ6VB9Cz9+djtnzCrg2tOmJu3+yQwQk4HY/LllzrZYdwMLgXJgM/BNVQ27PBcAEblJRIpEpKiysjJRZTdmxKhyOUkualxeBoe71CAONfR9iGtUdCQTYDWIPoo2M72+q5L65kgzk6ryL09tIdQR5v/7xJKkNC113j9pV4Z4pe46FusjwEZgErAMuFtE/C7PjWxUvU9VV6jqirFj3WV4NCaVRGsQhS5GMQGMzcs4LqNrdA7EpH7UIApzMzqbrnrLwWSO97ElE2nrUF7cFmlmWrv5EC9uO8y3Lp3Xp9Tj/ZHMAFEGxNZ9phCpKcS6EXhCI0qAPcACl+caY1yodpHqO9a4vMzjOqkrnLWd+9vWHV2C1JqY+m7Z1FFMHpXFs5vKqQ2G+Nc1Wzh5cj5/d0781fQSKZkBYj0wV0Rmikg6cC2wpssx+4GLAURkPDAfKHV5rjHGhWgNosBFJzVEEvZVNrYSDh+ttFfUt5Cb4et3kr3TZhQweVRWj9lkTXwiwuolE3ljVxV3PLGJuqY2/uuTS/ANwGiwpN1BVduBW4Hnge3Ao6q6VURuFpGbncN+BJwlIpuBl4HvqWpVd+cmq6zGjGRVgRB5Gb646yDEMy4vg/awUtt0dDZ1XxYKiueWC2fz4rfOS2p7+Uj2sSUTaQ8rz289zM3nz+akSQOz2lxS032r6lpgbZdt98b8XA5c5vZcY0zfVQfdTZKLOjpZrrVz5FNFfXO/RjBF+byeAfnEO1KdPDmfmYU5eARuvWjOgN3X1oMwZoSL5GFy10ENx06WW+gsR1BR33LMaCQzsESEP375dNK9Htc1wUSwAGHMCFcdCDF9TPzVy+IZl+fUIJyhrW0dYSoDrf0a4moSZ/Kogf/3tzqfMSNcdbCPNQj/sek2opPk+jPE1QxvFiCMGcHCYaUmGOrT6KHMNC95mT4qnQBxqB8ryZmRwQKEMSNYXXMbYcVVHqZYkZXlIoEhOkluojUxpRwLEMaMYH3JwxRrXF5mZ7qN6FKj/UnUZ4Y3CxDGjGB9zcMUNc5/bA0iJ93b63rWZuSxAGHMCFYddPIw9bkGkcGRhlZUtXOSnE1ySz0WIIwZwaqjNYg+9kGM92fS2h6moaWd8voW639IURYgjBnBqgOteARGZfctQIx1JstVNrZw6ARnUZvhywKEMSNYVTBEQU46Xk/fmoeik+XK61o40thqASJFWYAwZgSrDrQyxuU6ELGik+W2lNej2r+FgszwZwHCmBGsOtC3RH1R0XxMxQfqABvimqosQBgzgkUyufa9BpGb4SMrzcumsnqgf2tRm+HPAoQxI1hVoLXPI5ggkj10nD/j6CxqvzUxpSILEMaMUK3tHTS2tPd7FbdoM1N2uhd/lk2SS0UWIIwZoWo616LuexMTHB3JZJPkUldSA4SIrBKRnSJSIiJ3xNn/HRHZ6HxtEZEOESlw9u0Vkc3OvqJkltOYkai/k+SioiOZrP8hdSWt3igiXuAe4FKgDFgvImtUdVv0GFW9C7jLOf4K4B9UtSbmMheqalWyymjMSFbVz0R9UdEahM2iTl3JrEGsBEpUtVRVQ8DDwFU9HH8d8FASy2NMSonWIE60D8JqEKkrmQFiMnAg5nWZs+04IpINrAIej9mswAsiskFEburuJiJyk4gUiUhRZWVlAoptzMgQTdTX7xqE08RkCwWlrmQGiHi9WtrNsVcAb3ZpXjpbVU8FPgrcIiLnxTtRVe9T1RWqumLs2LEnVmJjRpDqQIgMn4ec9P4tcj9/fB7j8jJYNnVUgktmhotkjl0rA6bGvJ4ClHdz7LV0aV5S1XLn+xEReZJIk9XrSSinMSNSVSBEYW5Gv0cgjfNn8t4/XZLgUpnhJJk1iPXAXBGZKSLpRILAmq4HiUg+cD7wVMy2HBHJi/4MXAZsSWJZjRlxqoOt/UqzYUxU0moQqtouIrcCzwNe4AFV3SoiNzv773UOvRp4QVWDMaePB550Pvn4gD+p6nPJKqsxI1F/8zAZE5XU6ZGquhZY22XbvV1e/xb4bZdtpcDSZJbNmJGuOtDKvPF5g10MM4zZTGpjRiBVpSoY6vcQV2PAAoQxI1KgtZ1Qe9iamMwJsQxcJmW0tHVQtLeWN3dXcdJEP1csnTTYRaKtI8y+6iC7DgeoqG/BI+D1CB6P4JXId3+mj0tPmtCnVeGOptno3xwIY8ACxIBpaesgM61/49GHs7aOMPXNbQRa2gm0ttPofA+0thFo7SDY2k5TazuB1g6aQpF94/IyufWiORT0M4dQVDisbKtoYF1JFet2VbF+bw2t7WEgsuby6iUTk5KErrKxlac2HuRwQws+r4c0j+DzevB5hTSPh4aWNnYdDlBSGWBvVZD2cHfTg4565KYzOH3WGNdlODpJzmoQpv8sQCRZOKzcv66Unzz/IWfOHsO/rD6JOeNyezxHVSnaV0tWmpfFk/MHqKSJdaSxhd+s28OD7+wn0Nre6/E56V5yMnzkZPh4ruYQT35Qxj9evpBPLZ/S7Zt4fVMbv1lXyh/f3U+gpR115mGq834bViX63jt/fB7Xnz6dc+cWsrc6yL89vY1dRwIJ68RVVd4urebBd/fzwtZDtHUoWWle2sNh2jqODQBejzB9TDZzxubykUXjmTMulzlj85g8OpLzqCOshFXpCCsfHm7ki/+3vjMzq1tVnWk2rAZh+s8CRBIdbmjh9keLWVdSxRmzCnh/Xy2rfv46XzhrBt+4eC75WWnHHB8OKy9tP8wvX93NxgN1zBiTzavfuXCQSt8/+6ubuPf13Ty2oYz2jjAfPXkip88sIDfDF/nK9JGXkUZOhpdcJyBkpXnxxDSf7DjUwD89uYXvPraJxzaU8eOrFzNn3NE38oaWNh5Yt4ffrNtDY0s7l500ntlO0I1eJRpT5ozL5ezZhYzzH00XUVbbxL89vY11u6pOOEDUBkM8/n4Zf3p3P6VVQfKz0vj8mTO4buW0zg8C6rzZt4eVto4wGT4v6T533X9hJ9o1tLT1qVydTUxWgzAnwAJEkryw9RDfe3wTzW0d/Pjqk7lu5VSqgyF++sJOHnhzD09+cJBvXzafz5w2lbAqTxeXc+9ru/nwcIApo7M4d24hb+yqoqGlDX9mWu83BNo7wpTVNrOnOsjeqsjXnuomDtU3c9WyyXz1vFn4vMkZl7C9ooFfvbqbZzaV4/N4+OTyyXz1vNnMKMzp87UWTPDz56+eyaNFB/jPv+7go//9Bl89bzZfPHsGD727n/99o5QGJzDcdsk8Tprk79P1p4zOZsaYbN4sqeJL58zsc/miig/Ucf397xJobWf59NH87KI5XH7yxOOaEkUEn1fweelzM6Pf+RDR2NJ7LSxWtZPJ9USb6UxqswCRYM2hDv792W08+O5+Fk3y89/XntL5SbIwN4P//MQSrj99Oj98ehvff3Izv397L4HWdspqm5k/Po+ff2YZq5dM5M3d1byxq4pt5Q2c4aLt+cPDjXz8njdpCnV0bsvN8DGjMJv8rDTuen4nf9txhJ99einTx/T9TTuePVVB1m6u4NlNFWyraCAn3cuXz53F350zk/H+E0vw5vEI166cxiUnjefHz27n7ldKuPuVEgAuWTie2y6Ze0LNb2fPKeSpjeW0dYRJ60fQ3Fcd5Eu/Xc/onDT+fPOZLJzYtyDlVm66DxFoaO5jDSIYIi/TR4Yv9fq9TOJYgEigg3XNfOGB9yg5EuCm82Zx+2Xz4v6BLp6czyNfPYNnN1fwsxc/ZLw/kzuvWMRFC8Z1NrUscj4VbzlY7ypAvLGriqZQBz/6+GIWTMhjxpgcCnPTO9vvn9p4kH/+yxYu/+83+MEVJ/HpFVP71UHbNSgAnDJtFP/8sYV8avlU8rPd1XbcKszN4GefWcY1y6fw4vbDXH3KZJZMOfHkcWfPKeTBd/ezqayO5dML+nRudaCVLzzwHmFVfnfjSmaN7blP6UR4PEJuho+GPtYgqgKt1v9gTpirACEijwMPAH9V1XByizQ8dYSVf3hkI4fqW/jj353OOXMLezxeRFi9ZBKrl8QfalmYm8EEfyZbyxtc3b/4QB2T8jO54YzpcfdftWwyK2YU8O1Hi/ne45t5efsR/vMTJ7tOBX2gphcvBOQAAB2qSURBVIn/eHY7z209BBwNCh89eSKTRyV/QZmz5hRy1pye/0374sxZYxCBdbuq+xQgmkMdfPn3RVTUt/Cnr5ye1OAQ5c9M63sNIhDq90pyxkS5rUH8CrgR+B8R+TPwW1XdkbxiDT8PrNvDe3tquOuaJb0GB7cWT/az5WC9q2OLy+p6/WQ9eVQWD375dH6zbg93Pb+Tj/z8Db7zkXmsWjzxuA7zqOZQB796bTe/fm03HhFuu2Qun14xlUkDEBSSaXROOosn5fNmSRXfvGSuq3M6wso3Hv6AjQfq+NX1p/a55tFf/qy0PtcgqoOtzOxH/48xsVw1vqrqS6p6PXAqsBd4UUTeEpEbRSSxbQrD0M5Djdz1/E4uPWk81yyfkrDrLpqUz+7KAM0x/Qrx1AZD7KtuYqmLvP0ej/CV82bx1K1nMzYvg+89vpnT/v0lvvL7ItYUl9MUirwRqSrPbqrg4p++yv+8vIuPLJrA3759PrddMm/YB4eos+cU8sGBWoIuhuGqKneu2cqL2w7zr6tPYtXiiQNQwoi8TF+/RjH1d6EgY6Jc90GIyBjgc8ANwAfAg8A5wBeAC5JRuOEg1B7mHx7ZSF6mj//8xMkJnXi1aJKfsML2Qw2cOm10t8cVl9UBsHSq+07bhRP9rP3GORSX1fN0cTnPbCrnxW2HyUzzcPHC8VQHWnmntIaFE/38/NpTWDlzYD4tD6Sz54zh3td2897eGi6cP67HY+99rZQ/vLOPm86bxRfP7v/Ip/7wZ6ZRVtvk+viOsFLTFKLQmpjMCXLbB/EEsAD4A3CFqlY4ux4RkaJkFW44+O+XP2RbRQP33bA84Z2C0VE6Ww/W9xggNpXVIwIn93FUj4iwbOoolk0dxT9dvpCifbU8XVzO2s0VhFX5j6sXc+1p0/qU4mE4OW1GAek+D2/uquoxQLxVUsV/PbeD1UsmcseqBQNYwgh/lo/GCvdNTLVNIVT7v9SoMVFuaxB3q+rf4u1Q1RUJLM+wsmFfDb96dTefWj6FyxZNSPj1J+ZnUpCTzpaDPXdUFx+oY87YXPJczpeIx+MRVs4sYOXMAu68chHAiA0MUZlpXlZMH826kqoej/v5S7uYmJ/JTz619JgJfQPFn5nWpyYmmyRnEsXtAPCFItLZwC0io0Xk60kq07AQbG3nW48WMzE/ix9ccVJS7iEiLJrkZ0t59x3VqkpxWZ2r/ge3vB4Z8cEh6uw5hew41EiVM7Gsq3dLq3lvbw1fPW/WoOXS8mf6CLS2E3aRswmOTpKzRH3mRLkNEF9R1broC1WtBb6SnCINDz9eu539NU389NNLT+iTe28WTcrnw8ONhNrjjy4+WNdMVSCU0ACRSs52hs6+tbs67v67XymhMDeda1dOG8hiHcOflYYqNLroTAeoCkbzMFkNwpwYtwHCIzG9ryLiBXr97RORVSKyU0RKROSOOPu/IyIbna8tItIhIgVuzh1Mb+yq5MF39/Plc2a6msR2IhZP9tPWEUnaFk/xgUjtYumU4ZnUb7CdPDmfvEwfb+46vpnpg/21vLGriq+cO3i1B6Az1Uqjy2amzhqE9UGYE+Q2QDwPPCoiF4vIRcBDQI9rRDtB5B7go8BJwHUickxbjKreparLVHUZ8I/Aa6pa4+bcwfSXD8oZnZ3G7ZfNT/q9Fk9yOqq7aWYqLqsj3ethwYTkpHoY6bwe4azZY1hXUoXqsU0497xSwqjsNK7vZvLhQPFnRboKG5rd1SCqAyE8AqO6mdtijFtuA8T3gL8BXwNuAV4GvtvLOSuBElUtVdUQ8DBwVQ/HX0ck8PTn3AG141ADiyfnD8inymkF2eRl+LrtqC4+UMdJk/yus4Oa450zp5CDdc3srzk6lHRreT0vbT/Cl86eSW7G4GakiTZhuu2org62UpCTMSgd6mZkcTtRLqyqv1LVa1T1k6r6a1XtefYWTAYOxLwuc7YdR0SygVXA4/049yYRKRKRosrKSjePc0LaOsLsOhxIWnK2rjweYeEkf9waREdY2XywnmXW/3BCoik8Ykcz/fKV3eRl+PjCWTMGqVRHRZuY3KbbqArYWtQmMVwFCBGZKyKPicg2ESmNfvV2Wpxt3Q3DuAJ4U1Vr+nquqt6nqitUdcXYsWN7KdKJK60MEuoIs3BiYhaacWPxpHy2VTTQ0WUUS8mRAE2hjj5NkDPHm1WYw8T8TN50AkTJkUbWbqng82dN7zYFyUCKNjG5TfldHWi1Ia4mIdy2S/wfkXxM7cCFwO+JTJrrSRkwNeb1FKC8m2Ov5WjzUl/PHVDbnQymA1WDgEhHdUtbmNLKwDHbiw84M6gTkN00lYkIZ88p5K3d1YTDyi9f2U2mz8uXBnjGdHf62sRUEwxRYENcTQK4DRBZqvoyIKq6T1XvBC7q5Zz1wFwRmSki6USCwJquB4lIPnA+8FRfzx0M2ysaSPd6mD0AWTyjFjkd1V3nQ2wsq8Of6WNGgtZ3SGXnzCmkrqmNv245xFPF5Vx/+rQhMwooL7NvndS1TW2MTnDadZOa3AaIFhHxALtE5FYRuRroMXmNqrYDtxIZAbUdeFRVt4rIzSJyc8yhVwMvqGqwt3NdP1USbatoYM643H4tMtNfs8fmkOHzsLVLR3XxgUgGV+uMPHFnzY4MV77jiU14nYSGQ0Wa10N2utdVDaK9I0x9cxujs62JyZw4t8MzbgOygW8APyLSzPSF3k5S1bXA2i7b7u3y+rfAb92cOxTsONTIuQlK5+2Wz+th4cRjZ1S3tHWw81AjXz1/6LyRDWfj/JnMG5/Lh4cD3HDG9BNeES/R/JlpruZB1Dsd2VaDMInQ68dgZ07Cp1U1oKplqnqjM5LpnQEo35BSFWilsrGVkwaw/yFq8WQ/Ww82dKZb2FreQHtYrf8hgc6fN5Y0rwzJoJuX6XPVxFTb5AQIy+RqEqDXAOEMZ10eO5M6VQ1GB3XUokn5NLa2c8BJ+xztoLYhronzzUvm8ew3zmXK6OzBLspxIosG9V6DqG2KpNmwJiaTCG6bmD4AnnJWk4vtK3giKaUaogYzQBydUd3A9DE5FJfVMTE/k3FDrClkOMvN8DFv/MANX+4Lf6aPym4SCsaqDVqAMInjtqe1AKgmMnLpCudrdbIKNVRtr2hkvD+DgkGovs+bkIvPI51LkBYfqLPmpRTiz0pzNQ+izmliGmV9ECYBXNUgVPXGZBdkONhe0TAotQeADJ+XeePz2FLeQF1TiL3VTXz6tKm9n2hGhEgfRO9NTDVOE9NgfIgxI4/bFeX+jzgzmVX1Swkv0RAVag9TciTABb0sTZlMiyb5+duOIxSXRWoRy6wGkTIiiwa1o6o9Lmtb2xQi3RkWa8yJctvE9AzwrPP1MuAHAj2eMcKUHAnQHtYBTbHR1eLJ+VQHQ7yw9RAisNhSfKcMf1YaHWGlua3nFGh1wTZGZacldG10k7rcNjE9HvtaRB4CXkpKiYaoaAf1YAxxjVo8OXLvv3xwkNljczuTuJmR72jCvnay07v/s61pClnzkkmY/k4HngsM3hJbg2B7RQPpPg8zCwcvrcWCCX5EIBjqsA7qFNOZbqOXoa51TSHroDYJ4zaba6OINES/gKeJrBGRMrYfamD++Dx8A5hio6ucDB+znAC1zDK4phR/lruU35E8TFaDMInhtolpaA4OHyCqyvaKRi5ZOHgd1FGLJ+ezuzJoa1CnGH+mu5TftcGQzaI2CeO2BnG1k3U1+nqUiHw8ecUaWiobW6kJhobEsp4XzB/LtIJs5k9I6Zidctyk/FZV6potk6tJHLftJf+qqp2Z4lS1DvjX5BRp6Nk2iDOou7r6lCm8/t0LyfDZMMZUcnRd6u4DRENLOx1htSYmkzBuA0S84wZ3od4BtL2iERjcEUwmtXWOYuqhicnSbJhEcxsgikTkZyIyW0Rmicj/D2xIZsGGku0VDUzKzyTfqu5mkGSmeUn3eXpsYupM1Jdjv6cmMdwGiL8HQsAjwKNAM3BLsgo11Axmig1jovy9pPw+mofJahAmMdyOYgoCdyS5LENSS1sHpVVBPrJowmAXxaS4SLqN7msQNU4TU4EFCJMgbkcxvSgio2JejxaR512ct0pEdopIiYjEDTAicoGIbBSRrSLyWsz2vSKy2dlX5KacyVByJEBHWFkwiCk2jAHIy0rrsZPa1oIwiea2o7nQGbkEgKrWikiPkwKclejuAS4FyoD1IrJGVbfFHDMK+CWwSlX3x7nmhapa5bKMSTGURjCZ1ObP9PU4D6KuqQ2PHJ11bcyJctsHERaRztQaIjKDONldu1gJlKhqqaqGgIeBq7oc81ngCVXdD6CqR1yWZ8Bsr2ggM83DjDGDl2LDGOi9iam2KcSo7HQ8HkvUZxLDbYD4J2CdiPxBRP4AvAb8Yy/nTAYOxLwuc7bFmgeMFpFXRWSDiHw+Zp8CLzjbb+ruJiJyk4gUiUhRZWWly8dxb3tFA/Mn+PHaH50ZZP6snjupa5tCNknOJJTbTurnRGQFcBOwEXiKyEimnsR7R+1a6/ABy4GLgSzgbRF5R1U/BM5W1XKn2elFEdmhqq/HKdt9wH0AK1as6K1W0yfRFBuXn2wd1Gbw9VqDCFoeJpNYbhcM+jLwTWAKkQBxBvA2kSVIu1MGxC55NgUoj3NMlTNKKigirwNLgQ9VtRwizU4i8iSRJqvjAkQyHWpoob65zfofzJDgz0oj1B6mpa2DzLTjZ9LXNoWYMjp7EEpmRiq3TUzfBE4D9qnqhcApQG/tOeuBuSIyU0TSgWuBNV2OeQo4V0R8IpINnA5sF5EcEckDEJEc4DJgi8uyJkx0DYihkIPJmLxeEvbVNoUosElyJoHcDndoUdUWEUFEMlR1h4jM7+kEVW0XkVuB5wEv8ICqbhWRm53996rqdhF5DtgEhIH7VXWLiMwCnnRWxfIBf1LV5/r5jP0WTbFhQ1zNUOCPSdg3Ni/jmH2qaqm+TcK5DRBlzpDUvxDpD6jl+Oai46jqWmBtl233dnl9F3BXl22lRJqaBtW2igamjM6yldvMkBBN2BevBtHc1kGoPWyzqE1Cue2kvtr58U4ReQXIBwb8E/1AK6ttHtQV5IyJdXTZ0eM7qjtnUVsTk0mgPs+oUdXXej9qZKgJtjJzjHX6maGhpzUhLA+TSYbBWz9zGKgJhCjIyej9QGMGwNE1IY5vYrI0GyYZLEB0o6Wtg2CogzG59gdnhoZoE1NjnBqENTGZZLAA0Y2jf3AWIMzQkJ3uxesRa2IyA8YCRDcsQJihRkTI62ZNiGgT06gsq0GYxLEA0Y1qJ0CMsQBhhpDu0m3UBkP4M334vPYnbRLHfpu6URNsBawGYYYWf1b8lN+1TW2Mtt9Vk2AWILpRHYjWIGwUkxk68jLiLxoUTfVtTCJZgOhGTTCEzyOdQwuNGQr8Wb74TUxNIQos1bdJMAsQ3agJhhidk46TD8qYIcGfmRa/k9pSfZsksADRjepgyDqozZDjz0qLOw+izpqYTBJYgOhGTTBkHdRmyMnL9BEMddDeEe7c1toemdRpk+RMolmA6IYFCDMUHZ1NfbSZySbJmWSxANGN6kCrNTGZIcefdXzCPsvDZJLFAkQcbR1hGlraLVGfGXL8cVaVqw1GgsVoa2IyCWYBIo7aaJoNS9Rnhpi8OGtCWA3CJEtSA4SIrBKRnSJSIiJ3dHPMBSKyUUS2ishrfTk3WSzNhhmqOlN+WxOTGQBJmwUmIl7gHuBSoAxYLyJrVHVbzDGjgF8Cq1R1v4iMc3tuMkUT9dkfnBlqjq5LHa+T2pqYTGIlswaxEihR1VJVDQEPA1d1OeazwBOquh9AVY/04dyk6axBWBOTGWLiLTtaEwyRne4lM807WMUyI1QyA8Rk4EDM6zJnW6x5wGgReVVENojI5/twLgAicpOIFIlIUWVlZUIKXhOwRH1maMrNjDYxxXRSN4WstmuSIpmJhuLlqNA4918OXAxkAW+LyDsuz41sVL0PuA9gxYoVcY/pq5pgCBFrYjJDj9cj5GX4jqlB1DW1WfOSSYpkBogyYGrM6ylAeZxjqlQ1CARF5HVgqctzk6Y6GGJUVhpej+VhMkNPJN3G0RqETeo0yZLMJqb1wFwRmSki6cC1wJouxzwFnCsiPhHJBk4Htrs8N2nsD84MZXmZx2Z0tTxMJlmSVoNQ1XYRuRV4HvACD6jqVhG52dl/r6puF5HngE1AGLhfVbcAxDs3WWXtKpKozybJmaEpktE1dphrG6OtickkQVIXO1DVtcDaLtvu7fL6LuAuN+cOlJpgiDljcwfj1sb0yp/l42BdCwDtHWHqmy3Vt0kOm0kdR00wZLOozZDlzzya8rveqUlYDcIkgwWILjrCSm2TrQVhhq68zKOjmGqbonmY7PfVJJ4FiC7qmkKo2hwIM3T5s9JobG0n7HyYARuSbZLDAkQX0T84CxBmqPJnpqEKgVB7Z2JJCxAmGSxAdFEdiCbqs1FMZmiKJuxrbGm3PEwmqSxAdBFN1Gc1CDNUxab8rrEar0kiCxBdWKI+M9TFJuyrbQqR7vWQnW6J+kziWYDowlJ9m6HumCamYCQPk4ilhTGJZwGii5pgiLxMH+k++6cxQ9PRNSEiTUzWvGSSxd4Fu4ik2bA/ODN05UVTfje3OXmYrIPaJIcFiC5qgq32icwMaXkxq8pF8jDZ76tJDgsQXVQHQhTYEFczhKX7PGSleWlsidQgbBa1SRYLEF3UWBOTGQbyMn3UN7dZJleTVBYgYqhGUhdYoj4z1Pmz0jhY10xHWK2JySSNBYgYDS3ttHWo1SDMkOfP9LGvugmwIdkmeSxAxLBZ1Ga48GelUV7XDMDoHGtiMslhASJGTbAVsABhhr68zDTCGvnZlhs1yZLUACEiq0Rkp4iUiMgdcfZfICL1IrLR+fpBzL69IrLZ2V6UzHJGWaI+M1z4M48uBllgAcIkSdKWHBURL3APcClQBqwXkTWquq3LoW+o6upuLnOhqlYlq4xddabZsCq7GeL8WUd/R60PwiRLMmsQK4ESVS1V1RDwMHBVEu93wjoT9VkNwgxx0XQbHjk6s9qYREtmgJgMHIh5XeZs6+pMESkWkb+KyKKY7Qq8ICIbROSm7m4iIjeJSJGIFFVWVp5QgWuCIbLSvGRZZkwzxEWDwqjsdDweS9RnkiOZHz3i/dZql9fvA9NVNSAilwN/AeY6+85W1XIRGQe8KCI7VPX14y6oeh9wH8CKFSu6Xr9PaoKW+MwMD9EmJpskZ5IpmTWIMmBqzOspQHnsAaraoKoB5+e1QJqIFDqvy53vR4AniTRZJVV1MGTrQJhhIdpJbf0PJpmSGSDWA3NFZKaIpAPXAmtiDxCRCeIksheRlU55qkUkR0TynO05wGXAliSWFbBEfWb4iNYgbIirSaakNTGparuI3Ao8D3iBB1R1q4jc7Oy/F7gG+JqItAPNwLWqqiIyHnjSiR0+4E+q+lyyyhpVEwgxb3xesm9jzAmL1iAKbMSdSaKkDn9wmo3Wdtl2b8zPdwN3xzmvFFiazLLFuaetBWGGjegoJmtiMslkM6kdTaEOWtvDlurbDAv52Wn4M33MLMwZ7KKYEcwGUDtqOudA2CcyM/Rl+Lysu+MictLtT9gkj/12OaotUZ8ZZqLNTMYkizUxOToT9dkwV2OMASxAdDqaqM8ChDHGgAWITrYWhDHGHMsChKOmKUS610NuhnXLGGMMWIDoVBOI5GFyJucZY0zKswDhsER9xhhzLAsQDkvUZ4wxx7IA4bAahDHGHMsChMMChDHGHMsCBNDa3kGgtd3mQBhjTAwLEMTOgbBEfcYYE2UBgqOzqK2JyRhjjrIAQUwmVxvFZIwxnSxAYGk2jDEmnqQGCBFZJSI7RaRERO6Is/8CEakXkY3O1w/cnptI1bYWhDHGHCdpiYdExAvcA1wKlAHrRWSNqm7rcugbqrq6n+cmRE2wFa9HLL++McbESGYNYiVQoqqlqhoCHgauGoBz+6wmGGJ0dhoej+VhMsaYqGQGiMnAgZjXZc62rs4UkWIR+auILOrjuYjITSJSJCJFlZWV/SpodcAmyRljTFfJDBDxPo5rl9fvA9NVdSnwC+AvfTg3slH1PlVdoaorxo4d26+C2ixqY4w5XjIDRBkwNeb1FKA89gBVbVDVgPPzWiBNRArdnJtINcEQY2ySnDHGHCOZAWI9MFdEZopIOnAtsCb2ABGZIM4CDCKy0ilPtZtzE6naahDGGHOcpI1iUtV2EbkVeB7wAg+o6lYRudnZfy9wDfA1EWkHmoFrVVWBuOcmqZxctGAcp0wblYzLG2PMsCWR9+ORYcWKFVpUVDTYxTDGmGFDRDao6op4+2wmtTHGmLgsQBhjjInLAoQxxpi4LEAYY4yJywKEMcaYuCxAGGOMicsChDHGmLgsQBhjjIlrRE2UE5FKYF8/Ty8EqhJYnOHCnju12HOnFjfPPV1V42Y6HVEB4kSISFF3swlHMnvu1GLPnVpO9LmtickYY0xcFiCMMcbEZQHiqPsGuwCDxJ47tdhzp5YTem7rgzDGGBOX1SCMMcbEZQHCGGNMXCkfIERklYjsFJESEbljsMuTTCLygIgcEZEtMdsKRORFEdnlfB89mGVMNBGZKiKviMh2EdkqIt90to/0584UkfdEpNh57n9zto/o544SEa+IfCAizzivU+W594rIZhHZKCJFzrZ+P3tKBwgR8QL3AB8FTgKuE5GTBrdUSfVbYFWXbXcAL6vqXOBl5/VI0g7crqoLgTOAW5z/45H+3K3ARaq6FFgGrBKRMxj5zx31TWB7zOtUeW6AC1V1Wcz8h34/e0oHCGAlUKKqpaoaAh4GrhrkMiWNqr4O1HTZfBXwO+fn3wEfH9BCJZmqVqjq+87PjUTeNCYz8p9bVTXgvExzvpQR/twAIjIF+Bhwf8zmEf/cPej3s6d6gJgMHIh5XeZsSyXjVbUCIm+mwLhBLk/SiMgM4BTgXVLguZ1mlo3AEeBFVU2J5wZ+DnwXCMdsS4XnhsiHgBdEZIOI3ORs6/ez+5JQwOFE4myzcb8jkIjkAo8Dt6lqg0i8//qRRVU7gGUiMgp4UkQWD3aZkk1EVgNHVHWDiFww2OUZBGerarmIjANeFJEdJ3KxVK9BlAFTY15PAcoHqSyD5bCITARwvh8Z5PIknIikEQkOD6rqE87mEf/cUapaB7xKpP9ppD/32cCVIrKXSJPxRSLyR0b+cwOgquXO9yPAk0Sa0fv97KkeINYDc0VkpoikA9cCawa5TANtDfAF5+cvAE8NYlkSTiJVhd8A21X1ZzG7Rvpzj3VqDohIFnAJsIMR/tyq+o+qOkVVZxD5e/6bqn6OEf7cACKSIyJ50Z+By4AtnMCzp/xMahG5nEibpRd4QFX/Y5CLlDQi8hBwAZEUwIeBfwX+AjwKTAP2A59S1a4d2cOWiJwDvAFs5mib9PeJ9EOM5OdeQqRD0kvkg+CjqvpDERnDCH7uWE4T07dVdXUqPLeIzCJSa4BI98GfVPU/TuTZUz5AGGOMiS/Vm5iMMcZ0wwKEMcaYuCxAGGOMicsChDHGmLgsQBhjjInLAoQxQ4CIXBDNPGrMUGEBwhhjTFwWIIzpAxH5nLPOwkYR+bWTEC8gIj8VkfdF5GURGescu0xE3hGRTSLyZDQPv4jMEZGXnLUa3heR2c7lc0XkMRHZISIPSiokjDJDmgUIY1wSkYXAZ4gkRFsGdADXAznA+6p6KvAakRnqAL8HvqeqS4jM5I5ufxC4x1mr4Sygwtl+CnAbkbVJZhHJK2TMoEn1bK7G9MXFwHJgvfPhPotI4rMw8IhzzB+BJ0QkHxilqq85238H/NnJlTNZVZ8EUNUWAOd676lqmfN6IzADWJf8xzImPgsQxrgnwO9U9R+P2SjyL12O6yl/TU/NRq0xP3dgf59mkFkTkzHuvQxc4+Taj671O53I39E1zjGfBdapaj1QKyLnOttvAF5T1QagTEQ+7lwjQ0SyB/QpjHHJPqEY45KqbhORfyayYpcHaANuAYLAIhHZANQT6aeASGrle50AUArc6Gy/Afi1iPzQucanBvAxjHHNsrkac4JEJKCquYNdDmMSzZqYjDHGxGU1CGOMMXFZDcIYY0xcFiCMMcbEZQHCGGNMXBYgjDHGxGUBwhhjTFz/D6DZEIkYekr7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxcdb34/9d7luxJs7fpmrbpRkt3NitQFpUdBQQUUPQKrhf1qohe/Xm9V+8X70UEr2tZFBRRLIILggKyChS6QFvaQkvXtE2TNk0yWWYyy+f3xzmTTtMkzTZzzsy8n4/OY2bOnDPnc9LkPZ95n895f8QYg1JKqezhcboBSimlUksDv1JKZRkN/EoplWU08CulVJbRwK+UUllGA79SSmUZDfxKDUBEfiki3xnkujtF5NyRvo9SyaaBXymlsowGfqWUyjIa+FXas1MsXxGR9SLSISL3iMhYEXlcRAIi8pSIlCWsf4mIvCkiLSLyrIjMSXhtkYistbf7HZDXa18Xicjr9rYvicj8Ybb5BhHZJiLNIvInERlvLxcR+YGINIpIq31M8+zXLhCRTXbb9orIl4f1A1NZTwO/yhSXA+8BZgIXA48DXwcqsX7PbwIQkZnAg8AXgCrgr8CfRSRHRHKAR4FfAeXA7+33xd52MXAv8EmgAvg58CcRyR1KQ0XkbOD/AVcCNcAu4Lf2y+8FzrCPoxS4Cjhkv3YP8EljTDEwD/jHUParVJwGfpUp/s8Yc8AYsxd4AVhljFlnjAkBjwCL7PWuAh4zxjxpjAkDtwH5wLuAUwE/cIcxJmyMWQm8lrCPG4CfG2NWGWOixpj7gJC93VBcA9xrjFlrt+9rwGkiUguEgWJgNiDGmM3GmP32dmHgBBEpMcYcNsasHeJ+lQI08KvMcSDhcVcfz4vsx+OxetgAGGNiwB5ggv3aXnN05cJdCY+nAF+y0zwtItICTLK3G4rebWjH6tVPMMb8A/gR8GPggIisEJESe9XLgQuAXSLynIicNsT9KgVo4FfZZx9WAAesnDpW8N4L7Acm2MviJic83gN81xhTmnArMMY8OMI2FGKljvYCGGN+aIxZAszFSvl8xV7+mjHmUqAaKyX10BD3qxSggV9ln4eAC0XkHBHxA1/CSte8BLwMRICbRMQnIpcBJydsexfwKRE5xT4JWygiF4pI8RDb8BvgYyKy0D4/8N9YqamdInKS/f5+oAMIAlH7HMQ1IjLGTlG1AdER/BxUFtPAr7KKMeYt4Frg/4CDWCeCLzbGdBtjuoHLgOuBw1jnA/6QsO1qrDz/j+zXt9nrDrUNTwPfBB7G+pYxHbjafrkE6wPmMFY66BDWeQiA64CdItIGfMo+DqWGTHQiFqWUyi7a41dKqSyjgV8ppbKMBn6llMoyGviVUirL+JxuwGBUVlaa2tpap5uhlFJpZc2aNQeNMVW9l6dF4K+trWX16tVON0MppdKKiOzqa7mmepRSKsto4FdKqSyjgV8ppbJM0nL8InIvcBHQaIyJTyRRDvwOqAV2AlcaYw4P5/3D4TD19fUEg8HRaXCWy8vLY+LEifj9fqebopRKsmSe3P0lVk2T+xOW3QI8bYy5VURusZ9/dThvXl9fT3FxMbW1tRxdTFENlTGGQ4cOUV9fz9SpU51ujlIqyZKW6jHGPA8091p8KXCf/fg+4P3Dff9gMEhFRYUG/VEgIlRUVOi3J6WyRKpz/GPjswnZ99X9rSgiN4rIahFZ3dTU1N86yWllFtKfpVLZw7Und40xK4wxS40xS6uqjrn+QKmsc7A9xF/W73O6GSoDpDrwHxCRGgD7vjHF+x81LS0t/OQnPxnydhdccAEtLS1JaJHKdH9YW8/nfrOO/a1dTjdFpblUB/4/AR+1H38U+GOK9z9q+gv80ejAkyL99a9/pbS0NFnNUhmspTMMwKZ9bQ63RKW7pAV+EXkQayq7WSJSLyL/AtwKvEdEtgLvsZ+npVtuuYV33nmHhQsXctJJJ3HWWWfx4Q9/mBNPPBGA97///SxZsoS5c+eyYsWKnu1qa2s5ePAgO3fuZM6cOdxwww3MnTuX9773vXR1aU9O9S8QjAAa+NXIJW04pzHmQ/28dM5o7+vbf35z1P8YThhfwrcuntvv67feeisbN27k9ddf59lnn+XCCy9k48aNPcMh7733XsrLy+nq6uKkk07i8ssvp6Ki4qj32Lp1Kw8++CB33XUXV155JQ8//DDXXquz6am+tYfswL9fA78ambQo0pYOTj755KPGwP/whz/kkUceAWDPnj1s3br1mMA/depUFi5cCMCSJUvYuXNnytqr0k8gaKd6NPCrEcqIwD9QzzxVCgsLex4/++yzPPXUU7z88ssUFBSwfPnyPsfI5+bm9jz2er2a6lEDarNTPbsOdRIIhinO06us1fC4djin2xUXFxMIBPp8rbW1lbKyMgoKCtiyZQuvvPJKilunMlEgGCHXZ/3JvtXQ9++eUoORET1+J1RUVLBs2TLmzZtHfn4+Y8eO7XntvPPO42c/+xnz589n1qxZnHrqqQ62VGWK9lCYxZPLeHn7ITbtb2NpbbnTTVJpSgP/CPzmN7/pc3lubi6PP/54n6/F8/iVlZVs3LixZ/mXv/zlUW+fyiyBYIQZY4vY0tCmI3vUiGiqR6k0YIwhEIxQnOfjhPEleoJXjYgGfqXSQFc4SjRmKM7zc0JNCVsaAkSiMaebpdKUBn6l0kC7PaKnOM/HnJoSuiMxdhzscLhVKl1p4FcqDcSHchblWqke0PH8avg08CuVBuIXb5Xk+ZleVUSO16MneNWwaeBXKg3EyzUU5/nwez3MHFekPX41bBr4U6SoqAiAffv2ccUVV/S5zvLly1m9evWA73PHHXfQ2dnZ81zLPGeHeIG2ojxrBPYJNSVs2teGMcbJZqk0pYE/xcaPH8/KlSuHvX3vwK9lnrNDPNUTL9NwQk0Jhzq6aQyEnGyWSlMa+Ifpq1/96lH1+P/jP/6Db3/725xzzjksXryYE088kT/+8djpBnbu3Mm8efMA6Orq4uqrr2b+/PlcddVVR9Xq+fSnP83SpUuZO3cu3/rWtwCr8Nu+ffs466yzOOuss4AjZZ4Bbr/9dubNm8e8efO44447evan5Z/TXyBhVA/AnBo9wauGLzOu3H38FmjYMLrvOe5EOL//6QKuvvpqvvCFL/CZz3wGgIceeognnniCL37xi5SUlHDw4EFOPfVULrnkkn7ns/3pT39KQUEB69evZ/369SxevLjnte9+97uUl5cTjUY555xzWL9+PTfddBO33347zzzzDJWVlUe915o1a/jFL37BqlWrMMZwyimncOaZZ1JWVqblnzNAT6onxw788ZE9+9o4a1a/U1cr1Sft8Q/TokWLaGxsZN++fbzxxhuUlZVRU1PD17/+debPn8+5557L3r17OXDgQL/v8fzzz/cE4Pnz5zN//vye1x566CEWL17MokWLePPNN9m0adOA7XnxxRf5wAc+QGFhIUVFRVx22WW88MILgJZ/zgSBYISiXB8ej9WJKMnzM6k8X3v8algyo8c/QM88ma644gpWrlxJQ0MDV199NQ888ABNTU2sWbMGv99PbW1tn+WYE/X1bWDHjh3cdtttvPbaa5SVlXH99dcf930GOsmn5Z/Tn1WG+eg/1xNqStisQzrVMGiPfwSuvvpqfvvb37Jy5UquuOIKWltbqa6uxu/388wzz7Br164Btz/jjDN44IEHANi4cSPr168HoK2tjcLCQsaMGcOBAweOKvjWXznoM844g0cffZTOzk46Ojp45JFHOP3000fxaJWT2kORPgL/GHYc6qCzO+JQq1S6yowev0Pmzp1LIBBgwoQJ1NTUcM0113DxxRezdOlSFi5cyOzZswfc/tOf/jQf+9jHmD9/PgsXLuTkk08GYMGCBSxatIi5c+cybdo0li1b1rPNjTfeyPnnn09NTQ3PPPNMz/LFixdz/fXX97zHJz7xCRYtWqRpnQwRT/UkOmF8CcbAloYAiyeXOdQylY4kHcYBL1261PQe375582bmzJnjUIsyk/5M3evSH71IaUEO93385J5l9Yc7eff3nuE775/HtadOcbB1yq1EZI0xZmnv5ZrqUSoNxEsyJ5pQmk9Jnk9P8Koh01SPUmkgEIocM8euiFi1+Qc4wdsRihAIRvB4wOfx4BXpeZzn9/Q71FhltrQO/MYY/cUdJemQ8stmfY3qAesE729e3UU0ZvB6jv5bWLf7MNfd82pPnZ/eLlkwnjuvXqh/Q1kobQN/Xl4ehw4doqKiQn9xR8gYw6FDh8jLy3O6KaoP4WiMYDhGcW4fgX98CcGwVZu/rrqoZ/k7Te18/JevUV6Yw9cvmEPUGGIxQyRm3W9rbOd3q/dw2vQKPnTy5FQejnKBtA38EydOpL6+nqamJqebkhHy8vKYOHGi081QfehdriHRCXbphs3723oCf0NrkI/c8ypej3D/x0+mtrLwmO1iMUN9Syf/9ZdNnDatos91VOZK28Dv9/uZOnWq081QKumOzL7lP+a1uuoi/F5h0/42Ll4wntauMB+991VaOrv53SdP6zegezzCbR9cwPt+8Dz/9tDrPPTJ0/B5daxHttD/aaVcrs2uzFnUR48/x+ehrrqYTfvaCIaj3HDfarYfbOfn1y1l3oQxA75vzZh8vvOBE1m7u4WfPfdOUtqu3EkDv1IuN1CqB6x0z5v72rjpwXW8tquZ269cyLtnVPa5bm+XLBjPJQvGc8dTW1lfr/M6ZAsN/Eq5XHxUTkkfqR6wTvAebA/x900H+NZFJ3DxgvFDev//unQeVcW5fPF3r9PVHR1xe5X7aeBXyuXik7D0LtkQt3CSNRHPZ5ZP5/plQz/vNabAz20fXMA7TR1874ktw2+oShtpe3JXqWxxvFTPkillPP2lM5k2gpE5y+oq+fiyqdz7zx2cPbuaM2ZWDfu9lPtpj18pl+s97WJfplcVjfh6lpvPm8WM6iK+8ehGvaAvw2ngV8rlAqEIuT4POb7k/rnm+b18bNlUdjd3suNgR1L3pZzlSOAXkS+KyJsislFEHhQRvWRUqX70VaAtWZbVVQDwz20HU7I/5YyUB34RmQDcBCw1xswDvMDVqW6HUunCCvz9p3lG0+TyAiaW5fOiBv6M5lSqxwfki4gPKAD2OdQOpVyvvwJtySAivLuukpfeOUQ0pnn+TJXywG+M2QvcBuwG9gOtxpi/915PRG4UkdUislrr8ahs1t7H7FvJtKyukkAwwoa9rSnbp0otJ1I9ZcClwFRgPFAoItf2Xs8Ys8IYs9QYs7SqSoeWqeyVyhw/wLuma54/0zmR6jkX2GGMaTLGhIE/AO9yoB1KpQUr1ZOaHD9ARVEuJ9SU8OJWDfyZyonAvxs4VUQKxBp4fA6w2YF2KJUWrNm3Unut5btnVLJm12Et4ZChnMjxrwJWAmuBDXYbVqS6HUqlg1jM0B6K9DkJSzItq6ukOxrjtZ3NKd2vSg1HRvUYY75ljJltjJlnjLnOGBNyoh1KuV1HdwRjBr5qNxlOqi0jx+vRPH+G0it3lXKx49XpSZaCHB+LJpfqeP4MpYFfKReLl2TuaxKWZHt3XSVv7mujuaM75ftWyaWBXykXG0yBtmRZZk/m8tI72uvPNBr4lXKxNodSPQDzJ4yhONenef4MpIFfKReLT7Re4kDg93k9nDq9gn9uO5Tyfavk0sCvlIvFT+4W5aY+1QNWnn93cye7D3U6sn+VHBr4lXKxIzl+ZybLW1Zn5fn/qXn+jKKBXykXCwQjeAQKcryO7H96VSHjSvJ0WGeG0cCvlIu1h6zKnCOdVnG4RIRldZW8tO0gMS3TnDE08CvlYm0pLtDWl3fPqOBwZ5hN+9scbYcaPRr4lXKxVJdk7su7ptt5fk33ZAwN/Eq5WCAYpsThHv/YkjxmVBdpnj+DaOBXysXaQxFHyjX0tqyukld3NBMMa5nmTKCBXykXc0OqB+Ck2nJCkRjbGtudbooaBRr4lXIxtwT+uuoiAN5p0sCfCTTwK+Vi1kTrzub4AWorC/B6hK0HNPBnAg38SrlUMBylOxpzRY8/1+dlSnmBpnoyhAZ+pVwq4GCBtr5Mry5im6Z6MoIGfqVcysla/H2ZUV3EzoMdhKMxp5uiRkgDv1Iu1TP7VoonWu9PXXURkZhh16EOp5uiRkgDv1Iu5dR8u/2Jj+zRPH/608CvlEu5LdUzvcoK/DqyJ/1p4FfKpZycdrEvhbk+JpTm6wneDKCBXymXandZ4Ad7ZI+metKeBn6lXOrItIvuCfwzqot4p6lda/OnOQ38SrlUIBimIMeLz+ueP9O66iKC4Rh7W7qcbooaAff8RimljhKffctNdGRPZtDAr5RLuaVAW6K6Kg38mUADv1Iu5YZpF3srK8yhsiiHrY0Bp5uiRkADv1Iu5cYeP1jj+bXHn9408CvlUu0hdwb+GWOtwG+MjuxJVxr4lXKpQDBMsQtq8fdWV1VEWzBCUyDkdFPUMGngV8ql3JrqqasuBvQEbzpzJPCLSKmIrBSRLSKyWUROc6IdSrlVNGbo7I66YqL13nqGdGrphrTl1G/VncATxpgrRCQHKHCoHUq5z961BJsbAPcUaEs0tiSX4lyfFmtLYykP/CJSApwBXA9gjOkGulPdDqVc68Xbyd39GnC7K1M9IqI1e9KcE6meaUAT8AsRWScid4tIYe+VRORGEVktIqubmppS30qlnBJsw9fRQD5B10y72FudTsOY1pwI/D5gMfBTY8wioAO4pfdKxpgVxpilxpilVVVVqW6jUs4JWRdH1coBilw4qgesYm1NgRCtnWGnm6KGwYnAXw/UG2NW2c9XYn0QKKUAuq2edK00uDLVA4knePUK3nSU8sBvjGkA9ojILHvROcCmVLdDKdeye/xT0yHwa54/LTn1W/WvwAP2iJ7twMccaodS7tOT6mlw5XBOgIllBeT6PDqyJ0058ltljHkdWOrEvpVytVjsSKrH00CJC4dzAng9wrQqPcGbrvTKXaXcpPtIIJ0qDeT63PsnWqdDOtOWe3+rlMpGdpqnOaeGKmlFQu49eTqjuoi9LV10dkecbooaIg38SrmJ3ePfnTvTet683cHGDKyuughjYHtTh9NNUUOkgV8pN7F7+Nu8ddbz5nccbMzAdGRP+tLAr5Sb2IF/i0yznh9yb4+/tqIQr0c08KehQQV+Efm8iJSI5R4RWSsi701245TKOnbg3xspptlb6eoef47Pw5SKAp2GMQ0Ntsf/cWNMG/BeoApr3P2tSWuVUtnKDvwHu3Nozp0Ih9wb+MGalEV7/OlnsIFf7PsLgF8YY95IWKaUGi32yd3GkJ+W/Mmu7vGDNQ3jrkOddEdiTjdFDcFgA/8aEfk7VuD/m4gUA/o/rdRoC7UB0Njtp6NwCnQegq4WhxvVv7rqIiIxw65DOrInnQw28P8LVgXNk4wxnYAfLbOg1OgLtWO8uXTFfHSV1FrLXNzrnzW2BIBHX9/rcEvUUAw28J8GvGWMaRGRa4FvAK3Ja5ZSWSoUwORYwyQjpVOtZS4e2TOnppjLF0/kx8+8wxMbG5xujhqkwQb+nwKdIrIAuBnYBdyftFYpla1CAaJ+K/BTPg0QV/f4RYTvfmAeCyaV8qWHXuetBh3hkw4GG/gjxhgDXArcaYy5EyhOXrOUylLd7YR91hTUhYWFMMb9I3vy/F5WXLeEglwfN9y/mpZOnUnV7QYb+AMi8jXgOuAxEfFi5fmVUqMpFCDstXr8xbk+q9fv4h5/3NiSPH527RIaWoP864PriER17IebDTbwXwWEsMbzNwATgP9NWquUylahNoJeq8dfnOeHiumu7/HHLZlSxnc+MI8Xth7k1se3ON0cNYBBBX472D8AjBGRi4CgMUZz/EqNtlA7QYkHfh+UT4dgC3Q2O9ywwbly6SSuf1ctd7+4gz+srXe6Oaofgy3ZcCXwKvBB4EpglYhckcyGKZWVQgE67cBflOezevyQNr1+gH+/cA6nTavglj9sYONeHfznRoNN9fw71hj+jxpjPgKcDHwzec1SKkt1t9NBPiJQlGP3+CEt8vxxfq+HH1+zmDyfh/te2ul0c1QfBhv4PcaYxoTnh4awrVJqMKIRCHcSMHkU5fjweATKakE8adXjBygvzGFpbTnr9rj3quNsNtg5d58Qkb8BD9rPrwL+mpwmKZWluq0x8G0mz8rvA/hyYMyktOrxxy2eXMo/tjTS2hlmTIEOAnSTwZ7c/QqwApgPLABWGGO+msyGKZV1QlaBtpZIrjWiJy6NRvYkWjS5DIDX67XX7zaD7fFjjHkYeDiJbVEqu9klmQ9FcikrTAj85dOhfjUYA5I+RXEXTCpFBNbuOsyZM6ucbo5KMGDgF5EAYPp6CTDGmJKktEqpbGQH/sbuHMqrco4sr5huVe3sOAhF6RNAi3J9zBpbrHl+Fxow8BtjtCyDUqli5/gPBH3UFCQG/oT5d9Mo8IOV7vnL+n3EYsY6Wa1cQUfmKOUWdo+/IZRDeWLgL4/Pv5t+ef7Fk0sJBCNsP6izdLmJBn6l3MI+udsazaM0cRRM6RTw+NJyZE/8BO/aXZrucRMN/Eq5hd3jbyeP8sKEHr/XZwX/NOzxT6ssZEy+n3V7DjvdFJVAA79SbmEH/g7yKUtM9YB1gjcNe/wej7BwUqn2+F1GA79SbtEdIOrNI4qXssJegb98ujUTl+lrkJ27LZ5cxtuNAQLBsNNNUTYN/Eq5RShA2FcIQFnvK10rpkO4A9oPONCwkVk0uRRj4I09WrDNLTTwK+UWoQAhjx34j+nxp+/InoWT7Qu5dmue3y008CvlFqF2ujz5+Dxizb6VqKc887bUt2uESvL81FUVsU4Dv2to4FfKLUIBOiigtCAH6V2aYcwk8Oak5QlesPL86/a0YNLwHEUmcizwi4hXRNaJyF+caoNSrtIdoN3kUV7YRyVLj9cq0ZyGqR6w8vwtnWF2HOxwuikKZ3v8nwc2O7h/pdwlFKAtlkdp76GcceXToXl7ats0ShZPsS/k2q3DOt3AkcAvIhOBC4G7ndi/Uq4UCtASyzu6XEOiCjvwx2KpbdcoqKsqojjXp3l+l3Cqx38HcDPQ72+wiNwoIqtFZHVTU1PqWqaUU0LtNId7lWROVFEHkSC07kltu0aBxyMsnFyqPX6XSHngF5GLgEZjzJqB1jPGrDDGLDXGLK2qSq+KhEoNWSQE0RAHwznHXrUbVzXbum96K3XtGkWLJpXyVkMbHaGI003Jek70+JcBl4jITuC3wNki8msH2qGUe9gF2tpM/tF1ehJVxwN/ep4aWzSljJiBN3RGLselPPAbY75mjJlojKkFrgb+YYy5NtXtUMpVuuN1egY4uZtfBkXjoHFLChs2ehZNKgVgnaZ7HKfj+JVyA7tAW8Dk9z2cM656dtr2+EsLcphWVagneF3A0cBvjHnWGHORk21QyhXsVE8H+f33+AGq5lg5/jQc2QPWhVxrd+uFXE7THr9SbhCvxW/y+x/OCVaPP9wJrbtT1LDRtWhyKc0d3exu7nS6KVlNA79SbhBqAyDQVy3+RFVzrPs0zfMvjs/IpekeR2ngV8oNuq1UT1DyKc7z9b9e1Szrvik9A//MscUU5nh5adshTfc4SAO/Um5gp3p8+SV4PNL/evmlUFyTtoHf6xGWz6rm92vqufTH/+TxDfuJxvQDINU08CvlBvbJ3ZyCkuOvWzUbGtNzZA/A7Vct4P9ddiJtXWE+/cBa3vOD53jotT10R9LzhHU60sCvlBuEAnRJPmWFecdft3oOHHw7bUf25Pq8fOjkyTz9peX86MOLyPd7ufnh9ZzxP89w51NbefatRhrbgpoKSqIBkolKqZQJtdlDOQcYwx9XZY/sadkF5VOT37Yk8XqEi+aP58ITa3hh60F+8uw2fvDU2z2vVxTmMKemhDk1xSycVMb588YNnAZTg6aBXyk36G63a/EPMKInrtoe2dO0Ja0Df5yIcMbMKs6YWUVrZ5jNDW1s3h+/Bbjv5V3c9cIOLl4wnts+OJ9cn9fpJqc9DfxKuYAJBWiNHefirbj4yJ7GzTDr/OQ2LMXGFPg5dVoFp06r6FkWicZY8cJ2/ueJtzgYCPHzjyyhJG8Q34xUvzTHr5QLxLraCPQ3+1ZveWOgZMLIRvb86SZ47n+Gv30K+bwePrO8jtuvXMBrO5u58mcvc6At6HSz0poGfqVcIBoMHL9cQ6KqWcMf2ROLwYaVsPHh4W3vkMsWT+Te609iT3Mnl/3kJbY1BpxuUtrSwK+UG4QCtHOccg2JquIje6JD39fhHRDusLbvTq85cM+YWcXvPnkaoUiUy3/6Mmt2NTvdpBEJBMP8/Ll3uOnBdRzu6E7ZfjXHr5QLiH1yt3YwJ3fBqtkTCdoje6YNbWcNG6x7E4MDb8Kkk4e2vcPmTRjDHz69jI/+4lU+fNcq3jt3HAsnlbJwUilzx5eQ53f/yd/GtiD3/nMnD7yyi0AogtcjbG1s5zefOIWywf4OjIAGfqWcZgzecDvt5FM2mOGccHTNnqEG/gMbjzze/0baBX6AyRUFrPzUaXznsc28sv0Qf35jHwB+rzCnpoSFk0qpGZNPrs9Drt9Drs9rPfZ5yPV7yfFay3O8HvL8HnK8XnJ8HrwewecRPPa9N34TQcQagTQS25vaueuF7Ty8Zi+RWIzz59XwyTOn0dIZ5hP3r+bDd69KSfDXwK+U0yJBPCZCuykY3HBOSKjZsxlmXzC0/TVssD44Ohph/+tD29ZFKopy+cFVCwE40BZk3e4WXt/Twut7DrNyTT2d3cNIgx2HR6zrD0SOfBh4RBDo+WDw9LoX7HUEGtqC+L0ePrh0IjecPo3aysKe9777I0v5xP2ruebuVTyQ5OCvgV8pp9nlGjrJG/wwxbwSKJk4vCqdDRthyrug86DV488AY0vyOG/eOM6bNw6AWMzQHY0RCscIRaKEItZ9MOF5dyRmL7ced0diRI0hGo0RiRlixhCJGaJRQ8xA1Bhi9vL4Y2PAADFjPzbWugbrecxeZj02TCjL55pTplBVnHvMMZwxs4q7PrKUG+5fzbX3WMF/0Cf7h0gDv1JOs0syx3KKh3Zl6nBm4x4i95oAABl1SURBVOpshrZ6GDcPug7DS/9nTfTuOzYQpTOPR8jzeO18f/qM+T8zIfjHe/7JCP46qkcpp9mVOU1u8dC2q5oNB7cObWRPPL8/dh7ULIBYJK0LvmWiM2dWseK6JWxtbOeau1fR0jn6o3008CvlNLsWvy9/GIE/EoTDOwe/TYMd+MedaAV+yJh0TyZZPquaFdctYcfBDtbtGf3J6TXVo5TT4rX4C8YMbbvEmj0V0we3TcMGKBoLRdUQq4TcEg38LrV8VjUv3HwWFUWjn4bTHr9STrNP7uYOphZ/osSaPYN1YIPV2wfweGDcfA38LpaMoA8a+JVynLFP7uYWlQ5tw9xiGDNp8DV7It3WKKCx844sq1lg5f2jkaHtW6U1DfxKOSzc2QpAQXHZ0Deumj34IZ0H34ZY+EiPH6zAHwlar6msoYFfKYcFO1qJGqGkaIipHrCGdA62Zs+BhBO7cXqCNytlduD/553w6yucboVSA+ruaLUqcw7nSs2qORANQfOO46/bsAF8eVCecCK4cgb48qFh/dD3rdJWZgf+UADe+YeV21TKpSJdbQTIH3y5hkRVs637wVzI1bDBGgnkTRjM5/FaF3Npjz+rZHbgr5wJJmqVoVXKpWLBAB0mb3i1WXpG9hwnz2+MFfgT0zxxNQtg//q0nbxdDV1mB/6KOuteT1wpFzPBNrsy5zACf24RjJl8/B5/YD90NcPYfgJ/d0A7SFkkswN/5Qzr/uBWZ9uh1AAk3EEH+YzJH2ZNmerZ0PTWwOs09HFiN67nBG/6VupUQ5PZgT+3GIprNPArV/OFAwQ9BXiHUqAtUZU9smegsfjxk7dj5/ax/Rzw+DXPn0UyO/CD1evXVI9yMX+kg4ivaPhvUD0Hot0Dp2oObITSKVY55958OTD2BA38WSQLAv9MOLTVOrmllAvlRDuJ+guPv2J/Jiy17geaPL2/E7tx8RO8w/k7aW8c+jbKUZkf+CtmQLAVOpqcbolSxzKGfNM59JLMiapmwuyL4OUfW/X2e+vugEPvDBz4x823Tv621g9t37tehttmwpa/Dm075aiUB34RmSQiz4jIZhF5U0Q+n9Qd9pzg1XSPcqHuDjwYZCSBH+Csr1vXrbz8o2NfO7AJMMfp8VtTGA453fPsf1vv/eYfhradcpQTPf4I8CVjzBzgVOCzInJC0vamI3uUm9m1+L35wyjXkGjsXJj7AXjlZ9Bx8OjXDmyw15l37HaJ24tnaIF/18uw43nIK4W3/w7R8NDbrRyR8sBvjNlvjFlrPw4Am4EJSdthyUTrknQN/MqFgu3WJBv+/CHW4u/L8q9BpAv+ecfRyxs2Qu4YKJ3c/7Y5BVA5a2iB/7lbobAaLvw+hFph54vDa7dKOUdz/CJSCywCVvXx2o0islpEVjc1jSA/7/FAZZ2mepQrBdoOA5Az1Fr8famaCSdeCa/eDYEDR5Y3bLDKMshxhovWLBh84N/9Cmx/FpbdBLPOt2oAvaV5/nThWOAXkSLgYeALxpi23q8bY1YYY5YaY5ZWVVWNbGcVM6yRPbbtTe3cvPINguEhzFWqVBK0t1qBP694iLX4+3PmzdbQzhdvt57HYnDgzYHTPHE1C6C94egPjf48eysUVsHSj0NOIUw/2zrBq6Pn0oIjgV9E/FhB/wFjTPLPClXOhMO7IBwE4Ef/2MZDq+t5/m0d6aOc1WWnegqKhlGLvy8V02Hhh2H1vdC61xrbH+6wevzHE7+C93iVOnevgu3PwLtusoI+wKwLoK1eq3ymCSdG9QhwD7DZGHN7SnZaOQMw0Lyd5o5u/rJhPwBPb9bxx8pZwQ5rEpbiMaOQ448782ar5/3CbVaaBwYe0RMXX+d4pRueuxUKKuGkfzmybOZ5gOiwzjThRI9/GXAdcLaIvG7fLkjqHhOGdP5+9R66IzHmji/h6S2NxGL61VQ5p9sO/EUl5aP3pqWTYfFHYO2vrLy7eK2yDMeTVwLl0wbO8+951Sp1viyhtw9QVAWTToG3Hht5+1XSOTGq50VjjBhj5htjFtq35HYT7Cqdsaa3+fWqXZwytZwbTp/GwfYQb9S3JHXXSg0k0mWd3hozZpRSPXFnfNkanrn+d1aq0583uO1qFsI7z8CqFX3PY/HsrVBQASd94tjXZl9gfcNo2T2ytquky/wrd8HqmZRMpHHnRvY0d3HtqVNYPqsKr0c03aMcFQu2EcaLL7dgdN+4ZLx14hUGl9+PO+ebMH4RPP4V+PFJsGHlkTr9e16Dd54+OrefaNaF1v1bj4+s7SrpsiPwA1TOoGvfZiqLcnnf3HGUFuSwZEoZT20exAgGpZIlFKCTguMPtRyO0/8N8stgyrLBb1M+DT76Z7jmYcgphof/BVacaaV3nhugtw/WsOnKmbBF0z1ulzWBP1A8jarQbj500kRyfNZhnzunmi0NAeoPdzrcOpWtpLudkCc/OW9eVA1feguWXD+07URgxrnwyefhsrsg2AK/+gBsewpO+5w1+Ut/Zl0Au/4JXZpCdbOsCfyvtJZRJEGumZvbs+zcOWMB+McWTfcoZ/jC7YS8I6jMedwd5A7/24THA/OvhM+thvO+B3MugZNvGHib2RdCLAJbnxzePlVKZEXgD0WiPLzbyqGO6z5y4mlaVRHTKgt5cpOme5Qz/NER1uJPBV8unPopuOpX1uRGA5mw1CrjoKN7XC0rAv8TGxt4vdO++rdX6YZz5lSzansz7aEBZi9SKklGXIvfbTweq4TD1ichEnK6NaofWRH4f/XyLvLKJ2L8hXBo21GvnTNnLN3RGC/oVbwqxYLhKIUjrcXvRrMvtKqO7njB6ZaofmR84N+0r43Vuw5z7Wm1SB/TMC6dUsaYfD9P6bBOlWKHO7splODIa/G7zdQzwV+o6R4Xy/jA/+tVu8j1ebhiyUR7/t2je/w+r4ezZlXxzFuNRPUqXpVChzvCFNGFLz/DAr8/D+rOtsbzx68BUK6S0YE/EAzz6Lq9XLJgPKUFOdYY49bd0H308M1z5oyluaObdbsPO9RSlY0OdwQpkiC+0ajF7zazLoTAfquYW8tua+rHxi3WvL5711hX+AYadPIWh/icbkAyPbJuL53dUa49dYq1wC7dwKFtUDO/Z70zZ1Xh8whPbW5kae0o1kxRagCBNmuse25hBgb+me+zagT9+rLjr5s3xir6VlgJE0+C9/wneLzJb2MWy+jA3x6KcNq0ChZMsmudV8607g9tPSrwl+T5OWVaOU9vPsAt5892oKUqG3XYk7DkFY1SLX43KSiHa1fC4Z3gzQGPH7z2zeOHSBA6D0LHIfv+oPUN4eUfQU4RnPU1p48go2V04P/M8jo+feb0IwsqpgPS5zSM58wey3/+ZRO7D3UyuWKU66YM0oG2IGUFOT1XFqfK9qZ2/vdvb7GsrpIPnzwZjycJ5QPUMToDVo8/vygDe/xgTc4yFMbAo5+B574Hk06CunOT0y6V2Tl+AEm8atGfD6WT+pyGMX4VrxO1e5o7uvnK79/glP9+miX/9SQ3PbiOx9bvpyPJ1xYYY/jVK7u44Icv8OSmA3zj0Y1cteJltjW2J3W/qfbmvlb+vz9u5PEN+11Vhjteiz8jc/zDIWLN31t9Ajx8A7TscbpFGSuje/x9qpzZZ49/ckUBM6qLeGrzAT7+7qkpaYoxhofX7uW7j20iEIzwsWW1dIQiPLW5kT+9sY8cn4fT6yp539xxLJ9VRXXJIEvrDsKBtiA3r1zPc283cfqMSv73igW8sLWJ7zy2mQvufIHPnV3Hp86cnvJvH6Npf2sX3//72zy8th4B7n95F3NqSvjiuTN4zwljj+4UOCDcaQX+414Nm01yCuDK+2HFcvj99fCxx8GX43SrMk72Bf6KGbDrJWuYmefooHbuCWO56/ntHGgLUl2ce9zAEInGONTRjQhUFR1//UTbGtv5xqMbeGV7M0umlPHfHziRWeOKe9539a7D/O3NBv7+5gGetmsJ1VYUcPLUck6qLeeUqRVMKs8fVvD6y/p9fOPRjQTDUf7r0rlce+oURIQPLp3E8lnVfPvPb3L7k2/zl/X7uPXy+SyefHSt+GjM0NkdwesRCnLc9yvUHorw8+fe4a4XthOLwY2nT+NTZ07nubebuPPprdz4qzXMm1DCF8+dydmzqx37AIjX4h+w6Fk2qqyDS38Ev/8oPPlNOP97Trco44hJg8mRly5dalavXj06b/baPfDYv8EXNlppnwRrdh3m8p++BECe30N1cR7VxblUl+RSVZRLMBzjQCBIY1uIxkCI5o4Q8cxBvt/LxLJ8JpcXMKm8gIll+VSX5JGYLhesJ5v2t3LX8zvI83v42gVzuGrppH7z6sYY3tzXxivbD7FqRzOv7WympdMaAje2JJfZ40ooyvVRkOOlMNdHYa6XghwfeX4vxhiMAYMhZiBmDG/ubeOxDftZMKmU269cwPSqvoPO05ut1E9DW5DpVUV0dUfpCkfpCEUIRY6MzS4vzGFiWT4Ty/KZUJrPxLICxpbk4vN48HoEj0fwCHhFEBFixhCOxojGDOGoIRozRGIxPCL4PILP67Hvxdo+ISj39asqAoKV0hOBLQ0B7nxqKwfbQ1y8YDw3v28Wk8qPnLOJRGM8+vo+fvj0VnY3d7JgUimnTC0nGrPaEjPxe/B5hKI8H0W51q3Qvi/I8eL1WPvzitiP7ftebev9f5/oiV9/n68E74TPvwFltX3+P2S1J74Gr/wErrgX5l3udGtGX6Qbdr8M2560hrqOmQClU6zfhbIpUDbVKqs9go6JiKwxxiw9ZnnWBf4dL8B9F8F1jxxz8skYw5ObDrDzUEdPcG8MBGkKhGgKhMjze6kuyT3ygVCcS1VJHtFojD2Hu9jT3Nlzf7zaP+9fOJ5/v/AEqopzB1yvt1jMsK2pnVU7mnl1RzO7D3XQ0W0F5I5QhI7u6IAXouV4PXz2rDo+e9Z0fN6B0zjtoQg/+sc2dh7soCDXa3245PjIz7Eeh6OG+sNd1B/uZG9LF/WHu+iOOH/Bzkm1ZXz9gjksmtz/rFbhaIxH1u7lJ89u40Bb6Egg9whesT6wwtEYHaEI4Why/kau9z7Bf/jvh5t3WKNg1NEi3fDLC6FxE9zwDFTNHPl7xqJgYtboIie07LEC/danYMdzVmkLr32NUaDBGuGUKKcYrrwP6s4Z1u408McFGuD7s+D8/4FTPjk679mLMYaWzjAH248UqUr8Kef7vUf1Qkd736FIjFA4hnis3rBHrJ6zCD296mSIxQwHO0I0toUSes5W7zn+OP5NwG/36P1ej92rP/INIPEbgeHo38/EnrPBYP/r+WZTmOtj0aTSUU3fhCJROkJR2oMR2kMRusIRojHrmIwxRBOOtadtCc3u70+sdtNPqdv4A/hGk+ax+9NaDz8/wzoPMmEJhIMQ6bLuw50Q7QZ/gXUtQF6JfT8Gcksg3AXtDdDeaP3dtzdCR6MV+P2FkF8KeaXW+vn2fW6xtW1usfV+uSXWoJDuDmuOgWCrNT9B0H4carfa0d1hBfFu+3GsnwvTovZ0lqWToe49MOM9UHv6kXRfKACHd0HLLmso7OFdVpyqmN73+x1Hf4HffQnaZCsaa/1n9jGyZ7SICGWFOZQVpv6PWUTI83vJ86f+AhiPR+xvQ6N3EtoNcn1ecn1eykf7/3NvDLy5GvQHMmYiXPELeOxLsG+dFeR9eVYwLqq2esvhTisIt+w+Epij3dacw4XVUDwWisdBzQLr79+bY6/XeiSAt+yB4EYItVk3M8A3154PGvuDwl9gXYCWU3jk1t83isJqa5hq5Yy+Uzi5xdZUmUOZLnMYsi/wi1hX8PYxskeplAoF9MTuYEw7E/51iN/4w0H7YrFhdICMsT9M2qz/o3CHlXLJL7U6jRnwQZ19gR+sfNqO551uhcp2oXYdypks/hF86xQ50nOnZtSa5CbpO0h7JCpnQGAfvPJTqF9t9Q6USrVQQAO/ckR29vjrzoHX7oYnbrGee/wwdq518mjCYmusf1mtlUN0+CIfjLFOGvnynBuJoJKju91KISiVYtkZ+Mcvgn/bDG37rBKxe9fAvrWw/iFYfc+R9fwFCeNqa62TOV6/dXLImwNen3UfP9nkz7dGC/jzrW2Pus8/+kMk0g1t9dZJpdY91n1bvVW0quvw0bdYGBAorIKSGii2byXjrVEJIvbNY60nHuvm9YPHbmO8OJbX12s9OfLY67NPNubax2XfgzWBdjRstSUaPlJON/7ePT8T/5H7nvcfRcYcGSbjGeALaygAgQPWqI5Ag1UEzOO1/h8S/798+XYRMe+Rn5fHZz33+I/9WYzm8YTaoGjc6L2fUoOUnYEfrD/gMROs2wmXWMtiMWh+B5p32EOpEm47nrdO8oxE/ENAvNDRxNGDPMUacVBUZV20UT3bus8vs4J7uMtKT7Xth9a9UP8adB4aWXtSwZNQkdHjtUZLmJj1QRKLWvcmaq/cx4cRdqCPb5f4MzsqMOfZJ93E+tl2J6neUPwDrucDIvGDwtf3B0PPeM74h5b9vLX+SKlwpVIoewN/XzweK/9fOaPv12OxhB5vt30fsnrv4U4rOB9133lkvHHPrcvarrjGunJ4zCTrvmTi0EcLREJWzzYeGIkHSGMF02jY7ql3JzwO91ovYbto2HrPaLd9bx8bHF1SN/5tByAaf/+En0e021oeixz5ecXs5+KxPvg89i3+GI4ExZ77GEd9g0m8mdiR9kVDVpnfSLd13IXV1vC94nHWh2nxOGuZidljwBNuka6j25f4gRTtTnj/xJ9LX+vby/plfyDEP9QAJp0Ciz8ytP9zpUaBBv6h8HjAY6dC3MDnorYopdJGdo7qUUqpLKaBXymlsowGfqWUyjIa+JVSKsto4FdKqSzjSOAXkfNE5C0R2SYitzjRBqWUylYpD/wi4gV+DJwPnAB8SEROSHU7lFIqWznR4z8Z2GaM2W6M6QZ+C1zqQDuUUiorOXEB1wRgT8LzeuCU3iuJyI3AjfbTdhF5a5j7qwQOHnetzKPHnX2y9dj1uPs3pa+FTgT+vqpcHTM5nTFmBbBixDsTWd3X1GOZTo87+2TrsetxD50TqZ56YFLC84nAPgfaoZRSWcmJwP8aMENEpopIDnA18CcH2qGUUlkp5akeY0xERD4H/A3wAvcaY95M4i5HnC5KU3rc2Sdbj12Pe4jEmGPS60oppTKYXrmrlFJZRgO/UkplmYwO/NlSGkJE7hWRRhHZmLCsXESeFJGt9n2Zk21MBhGZJCLPiMhmEXlTRD5vL8/oYxeRPBF5VUTesI/72/byjD7uOBHxisg6EfmL/Tzjj1tEdorIBhF5XURW28uGfdwZG/izrDTEL4Hzei27BXjaGDMDeNp+nmkiwJeMMXOAU4HP2v/HmX7sIeBsY8wCYCFwnoicSuYfd9zngc0Jz7PluM8yxixMGLs/7OPO2MBPFpWGMMY8DzT3WnwpcJ/9+D7g/SltVAoYY/YbY9bajwNYwWACGX7sxhKfTd5v3wwZftwAIjIRuBC4O2Fxxh93P4Z93Jkc+PsqDTHBobY4YawxZj9YARKodrg9SSUitcAiYBVZcOx2uuN1oBF40hiTFccN3AHcDMQSlmXDcRvg7yKyxi5nAyM47kyebH1QpSFU+hORIuBh4AvGmDaRvv7rM4sxJgosFJFS4BERmed0m5JNRC4CGo0xa0RkudPtSbFlxph9IlINPCkiW0byZpnc48/20hAHRKQGwL5vdLg9SSEifqyg/4Ax5g/24qw4dgBjTAvwLNY5nkw/7mXAJSKyEyt1e7aI/JrMP26MMfvs+0bgEaxU9rCPO5MDf7aXhvgT8FH78UeBPzrYlqQQq2t/D7DZGHN7wksZfewiUmX39BGRfOBcYAsZftzGmK8ZYyYaY2qx/p7/YYy5lgw/bhEpFJHi+GPgvcBGRnDcGX3lrohcgJUTjJeG+K7DTUoKEXkQWI5VpvUA8C3gUeAhYDKwG/igMab3CeC0JiLvBl4ANnAk5/t1rDx/xh67iMzHOpnnxeq8PWSM+U8RqSCDjzuRner5sjHmokw/bhGZhtXLBys9/xtjzHdHctwZHfiVUkodK5NTPUoppfqggV8ppbKMBn6llMoyGviVUirLaOBXSqkso4FfqSQTkeXxSpJKuYEGfqWUyjIa+JWyici1dp3710Xk53YhtHYR+b6IrBWRp0Wkyl53oYi8IiLrReSReC10EakTkafsWvlrRWS6/fZFIrJSRLaIyAOSDQWFlGtp4FcKEJE5wFVYxbAWAlHgGqAQWGuMWQw8h3VVNMD9wFeNMfOxrhyOL38A+LFdK/9dwH57+SLgC1hzQ0zDqjujlCMyuTqnUkNxDrAEeM3ujOdjFb2KAb+z1/k18AcRGQOUGmOes5ffB/zerqcywRjzCIAxJghgv9+rxph6+/nrQC3wYvIPS6ljaeBXyiLAfcaYrx21UOSbvdYbqMbJQOmbUMLjKPq3pxykqR6lLE8DV9j1zuPzmU7B+hu5wl7nw8CLxphW4LCInG4vvw54zhjTBtSLyPvt98gVkYKUHoVSg6C9DqUAY8wmEfkG1ixHHiAMfBboAOaKyBqgFes8AFhlcH9mB/btwMfs5dcBPxeR/7Tf44MpPAylBkWrcyo1ABFpN8YUOd0OpUaTpnqUUirLaI9fKaWyjPb4lVIqy2jgV0qpLKOBXymlsowGfqWUyjIa+JVSKsv8/yzYkNIpXsciAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# chart accuracy and loss for train and validation sets\n",
    "\n",
    "print(modelfit.history.keys())\n",
    "#  acc\n",
    "#plt.plot(modelfit.history['accuracy'])\n",
    "plt.plot(modelfit.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "# plt.legend(['train', 'validation'], loc='upper left')\n",
    "#plt.legend(['validation'], loc='upper left')\n",
    "plt.show()\n",
    "# Loss\n",
    "plt.plot(modelfit.history['loss'])\n",
    "plt.plot(modelfit.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion matrix <a name='confusionmatrix' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEWCAYAAAB1xKBvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZgU1bnH8e877BJ2AREQjOIajYmKxCiiIKAS0SgGcUFFicZoTLw3EfXGLRqXuETjEqMERKOC+4ZAQDQoCogYIqLgCjKCwohrWGbe+8c5DTVNz0w1M9gw8/vMU890n6o6daqrut4+S1ebuyMiIlKVokIXQEREtgwKGCIikooChoiIpKKAISIiqShgiIhIKgoYIiKSSkEDhpk1MbMnzWylmY2rRj4nmNnEmixbIZjZeDMbupHr/sHMPjWzj2u6XHWJmV1qZvemXHaqmZ2+qctU08zsfTPrEx9faGZ3bWQ+b5hZrxot3Pq8f25mN9VwnqnLm3yNNoWsY3CumV29qbZVk1IFDDMbYmazzOxLMyuOF7YDamD7xwLtgTbuPmhjM3H3+9y9bw2Upxwz62VmbmaPZKV/P6ZPTZlPqouQux/m7qM3opydgfOB3dx9m3zXz3Nbo8zsD5sw/1PMbNqmyl/Kc/er3L3KoJfruLv77u4+tabLZGYNgYuB62oy35oqb7wuLK6BImXcCZxoZu1qMM9NosqAYWa/AW4CriJc3LcDbgMG1sD2uwBvu/vaGshrU/kE2N/M2iTShgJv19QGLKhOba8LsNzdl23EtutXY7ubPL+aZGb1Cl2GmrY5v97VMBCY7+4fFbog3wZ3/y8wHji50GWpkrtXOAEtgC+BQZUs04gQUJbE6SagUZzXC1hM+PS7DCgGTo3zLgNWA2viNoYBlwL3JvLuCjhQPz4/BXgX+AJ4DzghkT4tsd7+wExgZfy/f2LeVOAK4MWYz0Rg6wr2LVP+O4CzY1q9mPZ7YGpi2T8Di4DPgVeBA2N6/6z9fD1RjitjOb4Bdoxpp8f5twMPJfK/BpgMWFYZ+8T1y2L+o2L6kcAbwGcx310T67wP/A74N7Aq8/om5htwYzxmK+Ny3wOGx/1YHbf1ZEX5AT2Al+L2Xwd6ZZ1Xd8fz4SPgD/F13RX4L1Aa8/8sLt8EuB74IJZnGtAkzhsHfBzTXwB2T2xnVHwdnwG+AvrkOMbbA8/Hc2ES8BfKn4OV7UfyeO0ATAGWA58C9wEt47z/BR7O2u4twE0VnHfvAyOAeUAJ8HegcdY5+bu432MIH/wuAN6J2x8LtE7kd1J87ZYDF8X8+8R5l2bt7wGJ/V1EeG9Vdtwz+WzUdaCC/R8JXJx4Pho4Pz7uSLgm/CI+3xFYQXxfAAOAObH8LwF7Zr2umfI2ifmWAG8CvwUWZy37P4RzeiXwINAYaEr599uXwLbVOQZx/gnAc5VdjzeHqaqA0R9YS9YFJWuZy4GXgXZA23iQrkicKGvjMg2Aw4GvgVYVnKzZz7vGk6N+PFCfAzvHeR2IFwcSAQNoHU+Ck+J6x8fnbRJv8neAneJJMxW4uoJ960U40fcHXolphwMTgNMpHzBOBNrEbZ5PeDM3zrVfiXJ8COwe12lA+QvQVoRazCnAgYSLUKfKypl4vhPhAnlozPe3wEKgYeLNMAfoTLzwZuXXjxD0WhKCx65AhzhvFPCHHBe4dfkR3tTL42tVFMuxHGgbl38M+Gs8pu2AGcDPs49lIv9b42vTkRBY9mf9xeg0oBnrL1hzEuuNIrzZfxzL0TjHvk4Hbojr9yQEjnsTF6fK9iN5vHaM8xsR3gcvEAMC4Vz9ivUBpD7hwrl3BcfzfeA/8fVsTfhQ8Yes99Q1cVtNgPMI78FOMe2vwP1x+d0IF7Wecd4Ncf0NAgah9eALwnumAeF83quK457JZ6OvAzn2fyaJD6nxGGeC1BDC+/fBxLzH4+Mfxtd1P8J5MjSWsVGO8l5N+KDQKr5u/2bDgDGDEAxaE4LKmbnebzFto49BouwrNuXFviamqgLGCcDHVSzzDnB41sXm/cQL+w2JgBMPaI/sk7WC510pHzA+A44h6yJH+YBxEjAjx0XhlMSbPPnp5RfAsxXs27oTA1gA7Aw8EF+XcgEjx7olwPdz7VeiHJfnSDs98bw74dPTB8DxlWyr3AkM/B8wNvG8iPBJvlfizXBaJfkdQghWPYCirHmjyH3hOC3x/HfAmKxlJhDewO0JtZAmiXnHEz9dsWFtsSieQ9+v8mQOAc6BFomy3lPJ8tsR3rhNE2n/YP0FtML9yHW8spY7Cngt8Xw8cEZ8PACYV0m53idenOLzw4F3Esd6NYngR7iY9U4870CoEdQn1IQfSMxrGtfPFTBGAI9WUKaKjnsmn42+DuTY1gKgf+L5DoT3fhGhtv9z1r8vRwO/iY9vJwapxLpvAQflKO+7QL/EcqezYcA4MfH8WuCOXO+36h6DmNYNKK3qHC/0VFW7+XJg6yraSbclXNAyPohp6/Lw8n0UXwPfqWK7G3D3r4CfAWcCxWb2tJntkqI8mTJ1TDxPjiRKW54xwC+Bg4FHs2ea2flm9mYc8fUZodll6yryXFTZTHefQTixjVDFTavca+DuZXFbydegwm27+xRC08ytwFIzu9PMmlexzWR+XYBBZvZZZiI0dXSI8xoQjmFm3l8Jn0xz2ZrQFPBO9gwzq2dmV5vZO2b2OeFNnlmnyv0kvE4l8dzKSJ47le1HdlnamdkDZvZRLMu9WeUYTaiFEv+PqaRc2eXOfk994qHdO1nORxNlfJPQrNc+rrcur7ivyyvYZmdyvM4p1eR1oIRQawTA3d8hfELfi1DbfgpYYmY7AwcRagoQXofzs45X56xyJMubfI1znSf5XCeqewyaEWrDm7WqAsZ0QpvyUZUss4TwYmVsF9M2xleEppiMciN+3H2Cux9KeMPOB/6WojyZMlW3A20MoTbyjLt/nZxhZgcSPo0eR6hmtyQcfMsUvYI8K0rP5Hs2oQq7hNCslFa518DMjPDGSb4GlW7b3W92970JTWY7EdrhK1svmb6I8Mm8ZWJq6u5Xx3mrCP1GmXnN3X33CvL/lHAO7pBjm0MIHaR9CAG6a2aXU+5nMdDKzJom0rZLuR/Z/hi3tae7NycEhWQ5HgP2NLPvEWoY91VSLgjHK1mm5Hsqe58WAYdllbOxh07j4mReZrYVoakpl0Xkfp1zbTNbTV4H/k0455KeJ4yqbBj363lCJ3ErQnMohPJfmfU6bOXu9+fYRjGh+Sijc45lKpLrtajuMdiV0Ee2Was0YLj7SkJ16lYzO8rMtjKzBmZ2mJldGxe7H7jYzNqa2dZx+VTj2HOYA/Q0s+3MrAWhigyAmbU3syPjm3sV4RNHaY48ngF2ikOB65vZzwhtiE9tZJkAcPf3CJ9mLsoxuxmhaeMToL6Z/R5IfiJfCnTNZySUme1E6Aw+kdDM9lsz2yvl6mOBI8yst5k1IPSprCK0K6fZ9r5mtl9c9yvWd0Rn9uW7VWRxL/ATM+sXawGN41DETu5eTBhocL2ZNTezIjPbwcwOSuTfKQ6tzNSORgI3mNm2Mb8fmVkjwuu+ivBpbSvCSL7U3P0DYBZwmZk1jEPFf5JmP3Jk14zYUW9mHVkfYDPb+i/wEKHJa4a7f1hF8c42s05m1hq4kNDpWpE7gCvNrAtAfC9mRjE+BAwwswPia3o5Fb/v7wP6mNlx8b3TJnHOVXXca/I68AzhvZb0PKGG/0J8PhU4h9B8mTk3/wacGc9dM7OmZnaEmTVjQ2OBEWbWKh6vX+ZRvqVAm3iNyqjuMTiI0Gy5WavyAubuNwC/IYyL/oQQSX9J+MQE4aI2i/CpYC4wO6blzd0nEd4Y/yZ0uiYv8kWEC98SQrv+QYRP/Nl5LCd8gjufcCH5LTDA3T/dmDJl5T3N3XN9appAONhvE6ri/6V8FTfzpcTlZja7qu3EJsB7gWvc/XV3X0C4aIyJF8qqyvkWIdDcQviE/hPgJ+6+uqp1o+aEN18J60d2/CnOuxvYLVa9H8u1srsvInzyv5D158z/sv58OxloyPpRQA+xvplnCmF018dmljlm/0M4t2YSjv01Ma97Yvk+inm9nHL/koYQOklXAJfEPNPuR9JlhI7LlcDTwCM5lhkN7EHVzVEQAstEQpPku1T+nvoz8AQw0cy+ILwO+8V9eAM4O+ZXTHi9c36HIAaxwwnvnRWED3Dfj7OrOu41dh0AngR2MbNkU9LzhKCcCRjTCB8SMs9x91nAGYTm1BLCQI9TKtjG5YTX4T3gn4RzcFWawrn7fEKAfDe+HttSjWNgZo0Jr3ve38H6tmWGoonIJmZm2xGaUrdx988rWe59Qmf6P7+tsm1uzGw44Yuo531L2zsLGOzu2TWbb2Pb5wCd3T2fZueCqI1f+hHZ7MTmyN8QRstUGCwkcPc7N2X+ZtaB0MQ2nTBC6XxCzeRb5+63FGK7G0MBQ2QTi/1uSwnNZ/0LXBwJGhJG521PGLL7AOEOFlIJNUmJiEgqur25iIikssU3Sa359F1VkWQD++2x+d/HTb59s4unWdVLVS6fa06Drb9b7e1tTlTDEBEpIAu/jTHXzOaY2ayY1trMJpnZgvi/VWL5EWa20MzeMrN+ifS9Yz4Lzezm+IVdzKyRmT0Y018xs66JdYbGbSywFL/Fo4AhIpKPstL0U3oHu/te7r5PfH4BMNnduxHuUn0BgJntBgwm3IGhP3Cbrb9t/+2EOwt3i1NmgMUwwi1wdiTchfqamFdrwneP9iPct+6SZGDKRQFDRCQfpWvTTxtvIOu/yDea9bdnGkgYmr0q3n1iIdA9DhNu7u7TPYxkuidrnUxeDwG9Y+2jHzDJ3Ve4ewnh9v6VjuJTwBARyYN7WerJzIZb+LXSzDQ8V5aEb4i/mpjfPt5Gh/g/c3POjpS/i8TimNaR8t/gz6SXWyfeAHIl4V5WFeVVoS2+01tE5FtVVpZ60fgFxKq+hPhjd19i4SdaJ5nZ/EqWzdWJ7pWkb+w6OamGISKSDy9LP6XJLt6fzsNPLD9K6E9YGpuZMt9Kz/z88mLK31m3E+H+eospf/fdTHq5deJ96loQ7hVWUV4VUsAQEclHDXZ6xzvqNss8BvoSfm3xCcIPjhH/Px4fPwEMjiOftid0bs+IzVZfmFmP2D9xctY6mbyOBabEfo4JQN94x95WcdsTKiuvmqRERPKRsuaQUnvCDy9BuB7/w92fNbOZwFgzG0b4KedBEO58a2ZjCXdnXgucnbi9+1mEX0ZsQrh7duZ26XcT7nS9kFCzGBzzWmFmVxDuAg3hF0BXVFbYLf7WIPrinuSiL+5JLjXxxb1V77yc+prTaIceteqLe6phiIjkI49O79pGAUNEJB812yS1RVHAEBHJR37f4K5VFDBERPKhGoaIiKRSvVt+bNEUMERE8qFObxERSWP91x7qHgUMEZF8qA9DRERSUZOUiIikohqGiIikUrqm0CUoGAUMEZF8qElKRERSUZOUiIikohqGiIikooAhIiJpuDq9RUQkFfVhiIhIKmqSEhGRVFTDEBGRVFTDEBGRVFTDEBGRVNbqB5RERCQN1TBERCQV9WGIiEgqqmGIiEgqqmGIiEgqqmGIiEgqGiUlIiKpuBe6BAWjgCEikg/1YYiISCoKGCIikoo6vUVEJJXS0kKXoGAUMERE8lGHm6SKCl0AEZEtSllZ+ikFM6tnZq+Z2VPxeWszm2RmC+L/VollR5jZQjN7y8z6JdL3NrO5cd7NZmYxvZGZPRjTXzGzrol1hsZtLDCzoWnKqoAhIpIPL0s/pfMr4M3E8wuAye7eDZgcn2NmuwGDgd2B/sBtZlYvrnM7MBzoFqf+MX0YUOLuOwI3AtfEvFoDlwD7Ad2BS5KBqSIKGCIiefAyTz1Vxcw6AUcAdyWSBwKj4+PRwFGJ9AfcfZW7vwcsBLqbWQegubtPd3cH7slaJ5PXQ0DvWPvoB0xy9xXuXgJMYn2QqZAChohIPvJokjKz4WY2KzENz8rtJuC3QLI60t7diwHi/3YxvSOwKLHc4pjWMT7OTi+3jruvBVYCbSrJq1Lq9BYRyUceo6Tc/U7gzlzzzGwAsMzdXzWzXimys1ybqCR9Y9epkAKGiEg+am6U1I+BI83scKAx0NzM7gWWmlkHdy+OzU3L4vKLgc6J9TsBS2J6pxzpyXUWm1l9oAWwIqb3ylpnalUFVpOUiEg+amiUlLuPcPdO7t6V0Jk9xd1PBJ4AMqOWhgKPx8dPAIPjyKftCZ3bM2Kz1Rdm1iP2T5yctU4mr2PjNhyYAPQ1s1axs7tvTKuUahgF1PeYoTTdaiuKioqoV68eY0feXG7+ux8s4v+uvIF5by/k3OFDOXXIsdXe5urVqxlxxfXMe2sBLVs050+Xj6Bjh/YA7HngEXT7blcAOrRvy1+uvbTa25P8dNmhM1ffcfm65x27bMsd193FP/42bl1a1x2349IbL2SXPXbi1qv/xpg77q/2dhs0bMAVN1/MrnvuzGcln3PBz39P8eKP181v+p2tePiF+3hu/Atcc9GN1d7eFm3T33zwamCsmQ0DPgQGhc36G2Y2FpgHrAXOdvdM+9hZwCigCTA+TgB3A2PMbCGhZjE45rXCzK4AZsblLnf3FVUVTAGjwEbecjWtWrbIOa9F82Zc8OszmfLC9Lzz/ah4KRddeT2j/nJtufRHnppI82bfYfzYkTzzz6nccNtIrr9iBACNGjXk4dG35r8TUmM+eGcRxx96KgBFRUU8+9qjPDf+hXLLrCz5nGsvvomDD+uZd/4dOm3DZX++iOHHnFMu/ajjB/D5yi8YuP9g+g7sza8uPosLzrxk3fyzfncGr06fsxF7VAttgi/uuftUYpOQuy8Helew3JXAlTnSZwHfy5H+X2LAyTFvJDAyn3IWvEnKzHYxs9/FL5v8OT7etdDl2hy0adWSPXbdmfr1N4zrT06YwuDTf8UxQ8/msmtvpjRlR9yUf01n4OF9AOjb60BeeXUOXodv17w5637g3ix+/yOKFy8tl16y/DPmvT6ftWs2/F2Gw4/pyz3P3Mn9k/7ORdf+L0VF6d7ivfofwFNjw4fSyU9NZd8D9143b9c9d6bN1q14+fkZ1dibWqTM00+1TEEDhpn9DniA0GM/g1A9MuB+M7ugkGX7NpgZw399Eceddg7jHn8m9XrvvP8hz05+njF3XM/Do2+lqKiIpyY+l2rdZZ8sZ5t2WwNQv349vtN0Kz5b+TkQmquOO+1chpxxHpNfeCn/HZIa1W9gHyY89s/Uy2/frQt9j+zNaUeexfGHnkppaRmHHdM31bptt2nLx0tC32ppaSlffv4VLVu3wMz49SW/5KYrbtuofaiVSkvTT7VMoZukhgG7u/uaZKKZ3QC8QWjL20Acyzwc4Lbr/8DpJx+/qcu5SYy5/XratW3D8pLPOOO8C9m+S2f22WuPKtd7ZdYc5s1fyOBhvwJg1apVtG7VEoBzR1zOR0uWsmbtGoqXfsIxQ88G4MTjBnL0EX1z1ibiXQSY9PA9tGvbhkUfFTPs3Avo9t2ubNdp25raXclD/Qb16dnvx9xy1R2p1+l+wN7suufOjBkfvgPWqHEjSj4tAeBPI6+iY+cONGhYn206tuf+SX8H4P67xvHEg8+sOweS3J3jTjmaFydPZ+mSZRvMr6u8Dt9LqtABowzYFvggK70D5b/IUk5ybPOaT9/dYut97dq2AULTU++e+zN33lupAoa7c+Rhffj1WaduMO/mP/4eqLgPo327rfl42ads064ta9eW8uVXX9OiebNy5encsQP7/mBP5i94RwGjQH58SA/mz32bFfGCn4oZT44bz1+u+usGs/7ntAuBivswlhUvY5tt27Gs+BPq1avHd5o3ZWXJ5+yxz/f4wX7fZ9ApR9OkaRMaNGjA1199k1cgq3VqYVNTWoXuwzgPmGxm483szjg9S7h/yq8KXLZN6utv/stXX3297vFLM2avG6FUlR777MWkqdNYXvIZACs//4IlHy+tYq3g4AN68PgzoZlj4tR/sd/e38fMWPn5F6xevRqAks9W8trceezQdbs890pqSv+j+jDh0fTNUQAzpr1KnyN60apNqG02b9mMDp3ap1r3+QkvMuC4wwDoPaAXM6fNBuDisy/niH2OYUD3Qdx02a08Pe7Zuh0sYFPcS2qLUdAahrs/a2Y7EW5+1ZHQf7EYmJkYLlYrLV9Rwq8uvAKA0rWlHN63Fwf02IcHH30agJ8dfQSfLl/Bz4ady5dffU1RURH3jn2Mx+/7Kzts34VzzjiZ4eddRJmX0aB+fS76zS/YdpuqLw4/HdCPEVdcx2HHnUaL5s247rLQVfTuB4u4/NpbsCLDy5xhJx7HDtt32XQvgFSocZNG7NdzX6787XXr0o45eSAAD9/zOG3atubeZ++iabOmeFkZQ84YxLEHnch7b7/Pbdf8jdseuJGiImPt2lKuHnHDBp3muTx2/1Ncccv/8fhLD7Dys88Zcealm2r3tnx1uIZhW/oImS25SUo2nf32OLnQRZDN0OziabluiZGXr34/OPU1p+nlD1R7e5uTQvdhiIhsWWphU1NaChgiIvmow01SChgiInnQsFoREUlHNQwREUlFAUNERFKphbf8SEsBQ0QkD2l+q7u2UsAQEcmHAoaIiKSiUVIiIpKKahgiIpKKAoaIiKThpWqSEhGRNFTDEBGRNDSsVkRE0lHAEBGRVOpuF4YChohIPnxt3Y0YChgiIvmou/FCAUNEJB/q9BYRkXRUwxARkTRUwxARkXRUwxARkTR8baFLUDgKGCIieXDVMEREJBUFDBERSUM1DBERSaUuB4yiQhdARGRL4qWWeqqMmTU2sxlm9rqZvWFml8X01mY2ycwWxP+tEuuMMLOFZvaWmfVLpO9tZnPjvJvNzGJ6IzN7MKa/YmZdE+sMjdtYYGZD0+y7AoaISB68LP1UhVXAIe7+fWAvoL+Z9QAuACa7ezdgcnyOme0GDAZ2B/oDt5lZvZjX7cBwoFuc+sf0YUCJu+8I3AhcE/NqDVwC7Ad0By5JBqaKKGCIiOTByyz1VGk+wZfxaYM4OTAQGB3TRwNHxccDgQfcfZW7vwcsBLqbWQegubtPd3cH7slaJ5PXQ0DvWPvoB0xy9xXuXgJMYn2QqZAChohIHvKpYZjZcDOblZiGJ/Mys3pmNgdYRriAvwK0d/digPi/XVy8I7AosfrimNYxPs5OL7eOu68FVgJtKsmrUur0FhHJg3vlNYfyy/qdwJ2VzC8F9jKzlsCjZva9SrLLtWGvJH1j16mQahgiInmowT6M9Xm6fwZMJTQLLY3NTMT/y+Jii4HOidU6AUtieqcc6eXWMbP6QAtgRSV5VUoBQ0QkD2WllnqqjJm1jTULzKwJ0AeYDzwBZEYtDQUej4+fAAbHkU/bEzq3Z8Rmqy/MrEfsnzg5a51MXscCU2I/xwSgr5m1ip3dfWNapdQkJSKSh6o6s/PQARgdRzoVAWPd/Skzmw6MNbNhwIfAIAB3f8PMxgLzgLXA2bFJC+AsYBTQBBgfJ4C7gTFmtpBQsxgc81phZlcAM+Nyl7v7iqoKbCHYbLnWfPrulr0Dsknst8fJhS6CbIZmF0+r9tX+/b0OTX3N6TpnUo1Fl82BahgiInnYwj9jV4sChohIHmqwSWqLo4AhIpKHfIbV1jYKGCIieSitYvRTbaaAISKSB9UwREQkFfVhiIhIKholJSIiqaiGISIiqZSW1d07KilgiIjkQU1SIiKSSplGSYmISBoaVisiIqmoSWoL1mTbAwtdBNkM1d3PgLKpqUlKRERS0SgpERFJpQ63SClgiIjkQ01SIiKSikZJiYhIKmWFLkABKWCIiOTB6/AYPAUMEZE8rFWTlIiIpKEahoiIpKI+DBERSUU1DBERSUU1DBERSaVUNQwREUmjDv9CqwKGiEg+ylTDEBGRNHTzQRERSUWd3iIikkqZqUlKRERSKC10AQpIAUNEJA8aJSUiIqnU5VFSdffHaUVENoLnMVXGzDqb2XNm9qaZvWFmv4rprc1skpktiP9bJdYZYWYLzewtM+uXSN/bzObGeTebhY4WM2tkZg/G9FfMrGtinaFxGwvMbGiafVfAEBHJQ5mln6qwFjjf3XcFegBnm9luwAXAZHfvBkyOz4nzBgO7A/2B28ysXszrdmA40C1O/WP6MKDE3XcEbgSuiXm1Bi4B9gO6A5ckA1NFFDBERPJQlsdUGXcvdvfZ8fEXwJtAR2AgMDouNho4Kj4eCDzg7qvc/T1gIdDdzDoAzd19urs7cE/WOpm8HgJ6x9pHP2CSu69w9xJgEuuDTIUUMERE8lBq6SczG25msxLT8Fx5xqaiHwCvAO3dvRhCUAHaxcU6AosSqy2OaR3j4+z0cuu4+1pgJdCmkrwqpU5vEZE85PPFPXe/E7izsmXM7DvAw8B57v65Vfw9j1wzvJL0jV2nQqphiIjkoaaapADMrAEhWNzn7o/E5KWxmYn4f1lMXwx0TqzeCVgS0zvlSC+3jpnVB1oAKyrJq1IKGCIieXBLP1Um9iXcDbzp7jckZj0BZEYtDQUeT6QPjiOftid0bs+IzVZfmFmPmOfJWetk8joWmBL7OSYAfc2sVezs7hvTKqUmKRGRPNTgvaR+DJwEzDWzOTHtQuBqYKyZDQM+BAYBuPsbZjYWmEcYYXW2u2e+eH4WMApoAoyPE4SANMbMFhJqFoNjXivM7ApgZlzucndfUVWBLQSbLVf9hh237B2QTaLufrVKKrNm9UfVPjVu6Xxi6mvOOYvurVWnomoYIiJ50K1BREQkFd3eXEREUlHAEBGRVOpyp6kChohIHtSHISIiqegHlEREJJWyOtwopYAhIpIHdXqLiEgqdbd+oYAhIpIX1TBERCSVtVZ36xgKGCIieai74UIBQ0QkL2qSEhGRVDSsVkREUqm74UIBQ0QkL2qSEhGRVErrcB1DAUNEJA+qYYiISCquGoaIiKRRl2sYRYUuQF1XVFTEzBkTePzR0RvMO6jnj1j+yZvMmjmRWTMncvFF51V7ew0bNuQf993O/HnTeGnak3Tp0gmA7bbryCsvj2fWzIm8PmcKw884qdrbkvzttNMO6wGU/tcAAAv1SURBVI73rJkTWf7pfM495/QNluvZ80fMmjmROXOmMPmfD1V7uw0bNuS++27nzXnTeLGC82KOzgsgDKtNO9U2qmEU2LnnnM78+Qto3qxZzvnTps1g4NFD8863S5dOjLzrRnofOqhc+mmnHk9JyUp22e0AjjvuSP541UUMOeEsiouXcWDPgaxevZqmTbfi9dem8ORTEykuXrpR+yUb5+2332GfffsC4cPEB++/ymOPjy+3TIsWzbnllqsYMOAEFi1aQtu2bVLn36VLJ+6+60b65DgvPitZya7xvLjqqos4Icd5MUfnRS0MA+mphlFAHTt24PDDejNy5P15rztkyE+Z/uJTzJo5kdtuvYaionSH8sif9GXMmHEAPPzw0xxy8AEArFmzhtWrVwPQqFGj1PnJpnPIIQfw7rsf8OGHH5VLP37w0Tz22HgWLVoCwCefLF83b8iQn/LSRpwXP9F5kdpaPPVU2+joF9AN11/GBSP+QFlZxa2iPXrszauzJvHUE2PYbbedANhllx05btCRHHjQUeyzb19KS0sZMuSnqba5bcdtWLQ4XGhKS0tZufJz2rRpBUCnTtsy+9VJvP/uTK770611+lPk5uBnxw3kwQcf2yC9W7fv0qplC/45aRyvvDyeE088FgjnxaBBR9JzE50X7707kz/pvMDz+KttNtsmKTM71d3/XsG84cBwAKvXgqKipt9q2WrCEYf3YdmyT5n92lwO6vmjnMvMfm0u392xO1999TWH9T+Eh8eNZNfdD+CQgw/ghz/Yg5enPwNAkyaN+eSTTwF4aNxddO26HQ0bNmC7zh2ZNXMiALfcchej7xmL2YY/SOzxvF68eAk/3PtQOnRozyMP3c3DjzzNsmWfboK9l6o0aNCAAQP6ctHFf9xgXv369fjhD/ekb7/jaNKkMf964UleeWX2BudF4yaNWRbPi3Hj7mL7rtvRoJrnxcM6L+p0p/dmGzCAy4CcAcPd7wTuBKjfsOMWGcb3338ffjKgL4f1P4TGjRvRvHkzRo+6maGnnLtumS+++HLd4/HPTuGWm6+iTZtWmBlj7h3HRRdfvUG+xw4KHaQV9WF8tLiYzp225aOPiqlXrx4tWjRnxYqScssUFy/ljXlvc8AB+/HII0/X5G5LSv37H8xrr83NeWFe/FExny5fwddff8PXX3/DtGkvs+eeu607Ly7OcV4MSpwXufow0p4X83Re1MqaQ1oFbZIys39XMM0F2heybJvaRRdfTdfv7sOOO/XghBN/wXPPvVguWAC0b9923eN999mLoqIili8vYcpz0/jp0QPWdXa2atWS7bbrmGq7Tz41kZNOCheLY445guemvgiE/pTGjRsD0LJlC/bff1/efvudau+nbJyf/eyonM1RAE8+OYEDfrwf9erVo0mTxuzb/QfMn7+gWufFUynPix/pvKAsj6m2KXQNoz3QDyjJSjfgpW+/OIWXGbZ459/GcMxPj+DnPz+ZtWtL+e83/+WEE38BwJtvLuD3l17L+Gfup6jIWLNmLeeee9EGnaO5jPz7A4wedTPz502jpOQzhsQ8d91lR6699ve4gxnccMMd/Oc/8zfdjkqFmjRpTJ/ePfnFL363Li15Xsyfv5AJE59j9ux/UlZWxt9H3s8bb7wFwCXVOC9GjbqZN+N5kTnXdtllR65LnBc36ryg1OtuDcO8gDtvZncDf3f3aTnm/cPdh1SVx5baJCWb1oYt8iKwZvVH1T41hnQ5OvU15x8fPFqrTsWC1jDcfVgl86oMFiIi37a63IdR6CYpEZEtSm3sm0hLAUNEJA+18ZYfaSlgiIjkQU1SIiKSSl0eJaVbg4iI5KEm71ZrZiPNbJmZ/SeR1trMJpnZgvi/VWLeCDNbaGZvmVm/RPreZjY3zrvZ4lf3zayRmT0Y018xs66JdYbGbSwws1R3OFXAEBHJQw1/cW8U0D8r7QJgsrt3AybH55jZbsBgYPe4zm1mVi+uczvhdknd4pTJcxhQ4u47AjcC18S8WgOXAPsB3YFLkoGpIgoYIiJ5qMmbD7r7C8CKrOSBQOYHckYDRyXSH3D3Ve7+HrAQ6G5mHYDm7j7dwxfr7slaJ5PXQ0DvWPvoB0xy9xXuXgJMYsPAtQH1YYiI5OFbGCXV3t2LAdy92MzaxfSOwMuJ5RbHtDXxcXZ6Zp1FMa+1ZrYSaJNMz7FOhVTDEBHJg7unnsxsuJnNSkzDq7HpXN8a90rSN3adCqmGISKSh9I8ahjJO2vnYamZdYi1iw7Aspi+GOicWK4TsCSmd8qRnlxnsZnVB1oQmsAWA72y1plaVcFUwxARycO38JveTwCZUUtDgccT6YPjyKftCZ3bM2Lz1Rdm1iP2T5yctU4mr2OBKbGfYwLQ18xaxc7uvjGtUqphiIjkoSZv2Gpm9xM+6W9tZosJI5euBsaa2TDgQ2BQ3O4bZjYWmAesBc5299KY1VmEEVdNgPFxArgbGGNmCwk1i8ExrxVmdgUwMy53ubtnd75vWN5C3q22JuhutZJLrbpFqNSYmrhb7cGdDk19zXlu8aRadSqqhiEikgfdGkRERFKpy7cGUcAQEcmD7lYrIiKpKGCIiEgqW/pAoepQwBARyYNqGCIikopGSYmISCqlXnd/1VsBQ0QkD+rDEBGRVNSHISIiqagPQ0REUilTk5SIiKShGoaIiKSiUVIiIpKKmqRERCQVNUmJiEgqqmGIiEgqqmGIiEgqpet+RrvuUcAQEcmDbg0iIiKp6NYgIiKSimoYIiKSikZJiYhIKholJSIiqejWICIikor6MEREJBX1YYiISCqqYYiISCr6HoaIiKSiGoaIiKSiUVIiIpKKOr1FRCQVNUmJiEgq+qa3iIikohqGiIikUpf7MKwuR8vaxsyGu/udhS6HbF50XkhNKSp0AaRGDS90AWSzpPNCaoQChoiIpKKAISIiqShg1C5qp5ZcdF5IjVCnt4iIpKIahoiIpKKAISIiqShg1BJm1t/M3jKzhWZ2QaHLI4VnZiPNbJmZ/afQZZHaQQGjFjCzesCtwGHAbsDxZrZbYUslm4FRQP9CF0JqDwWM2qE7sNDd33X31cADwMACl0kKzN1fAFYUuhxSeyhg1A4dgUWJ54tjmohIjVHAqB0sR5rGS4tIjVLAqB0WA50TzzsBSwpUFhGppRQwaoeZQDcz297MGgKDgScKXCYRqWUUMGoBd18L/BKYALwJjHX3NwpbKik0M7sfmA7sbGaLzWxYocskWzbdGkRERFJRDUNERFJRwBARkVQUMEREJBUFDBERSUUBQ0REUlHAkIIws1Izm2Nm/zGzcWa2VTXyGmVmx8bHd1V240Uz62Vm+yeen2lmJ2/stkXqEgUMKZRv3H0vd/8esBo4Mzkz3oE3b+5+urvPq2SRXsC6gOHud7j7PRuzLZG6RgFDNgf/AnaMn/6fM7N/AHPNrJ6ZXWdmM83s32b2cwAL/mJm88zsaaBdJiMzm2pm+8TH/c1stpm9bmaTzawrITD9OtZuDjSzS83sf+Lye5nZy3Fbj5pZq0Se15jZDDN728wO/FZfHZHNRP1CF0DqNjOrT/gdj2djUnfge+7+npkNB1a6+75m1gh40cwmAj8Adgb2ANoD84CRWfm2Bf4G9Ix5tXb3FWZ2B/Clu/8pLtc7sdo9wDnu/ryZXQ5cApwX59V39+5mdnhM71PTr4XI5k4BQwqliZnNiY//BdxNaCqa4e7vxfS+wJ6Z/gmgBdAN6Anc7+6lwBIzm5Ij/x7AC5m83L3S34UwsxZAS3d/PiaNBsYlFnkk/n8V6JpuF0VqFwUMKZRv3H2vZIKZAXyVTCJ84p+QtdzhVH37dkuxTD5Wxf+l6H0jdZT6MGRzNgE4y8waAJjZTmbWFHgBGBz7ODoAB+dYdzpwkJltH9dtHdO/AJplL+zuK4GSRP/EScDz2cuJ1GX6pCSbs7sIzT+zLVQ/PgGOAh4FDgHmAm+T48Lu7p/EPpBHzKwIWAYcCjwJPGRmA4FzslYbCtwRh/i+C5y6KXZKZEulu9WKiEgqapISEZFUFDBERCQVBQwREUlFAUNERFJRwBARkVQUMEREJBUFDBERSeX/ATpzWSZKCnDUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "cfmap=metrics.confusion_matrix(y_true=test['target'],  # True labels\n",
    "                         y_pred=test[\"predround\"])\n",
    "\n",
    "label = [\"0\", \"1\"]\n",
    "sns.heatmap(cfmap, annot = True, xticklabels = label, yticklabels = label)\n",
    "plt.xlabel(\"Prediction\")\n",
    "plt.title(\"Confusion Matrix for streetcar delay prediction (weighted)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['loss', 'accuracy', 'accuracy_1']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kaggle submission that was used as input for this notebook\n",
    "https://www.kaggle.com/knowledgegrappler/a-simple-nn-solution-with-keras-0-48611-pl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary\n",
    "This notebook shows methods for dealing with structured data in the context of a neural network.\n",
    "\n",
    "# Author\n",
    "\n",
    "Mark Ryan is a manager at Intact Insurance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "try_tf2",
   "language": "python",
   "name": "try_tf2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
